Add SBOM, symbols, traces, and VEX files for CVE-2022-21661 SQLi case
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Created CycloneDX and SPDX SBOM files for both reachable and unreachable images.
- Added symbols.json detailing function entry and sink points in the WordPress code.
- Included runtime traces for function calls in both reachable and unreachable scenarios.
- Developed OpenVEX files indicating vulnerability status and justification for both cases.
- Updated README for evaluator harness to guide integration with scanner output.
This commit is contained in:
master
2025-11-08 20:53:45 +02:00
parent 515975edc5
commit 536f6249a6
837 changed files with 37279 additions and 14675 deletions

View File

@@ -1,7 +1,9 @@
using System.Text.Json;
using Microsoft.Extensions.Options;
using StellaOps.Aoc;
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
using System.Diagnostics;
using System.Text.Json;
using Microsoft.Extensions.Options;
using StellaOps.Aoc;
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
using StellaOps.Ingestion.Telemetry;
namespace StellaOps.Excititor.Core.Aoc;
@@ -21,15 +23,35 @@ public sealed class VexRawWriteGuard : IVexRawWriteGuard
_options = options?.Value ?? AocGuardOptions.Default;
}
public void EnsureValid(RawVexDocument document)
{
ArgumentNullException.ThrowIfNull(document);
using var payload = JsonDocument.Parse(JsonSerializer.Serialize(document, SerializerOptions));
var result = _guard.Validate(payload.RootElement, _options);
if (!result.IsValid)
{
throw new ExcititorAocGuardException(result);
}
}
}
public void EnsureValid(RawVexDocument document)
{
ArgumentNullException.ThrowIfNull(document);
using var guardActivity = IngestionTelemetry.StartGuardActivity(
document.Tenant,
document.Source.Vendor,
document.Upstream.UpstreamId,
document.Upstream.ContentHash,
document.Supersedes);
using var payload = JsonDocument.Parse(JsonSerializer.Serialize(document, SerializerOptions));
var result = _guard.Validate(payload.RootElement, _options);
if (!result.IsValid)
{
var violationCount = result.Violations.IsDefaultOrEmpty ? 0 : result.Violations.Length;
var primaryCode = violationCount > 0 ? result.Violations[0].ErrorCode : string.Empty;
guardActivity?.SetTag("violationCount", violationCount);
if (!string.IsNullOrWhiteSpace(primaryCode))
{
guardActivity?.SetTag("code", primaryCode);
}
guardActivity?.SetStatus(ActivityStatusCode.Error, primaryCode);
throw new ExcititorAocGuardException(result);
}
guardActivity?.SetTag("violationCount", 0);
guardActivity?.SetStatus(ActivityStatusCode.Ok);
}
}

View File

@@ -13,5 +13,6 @@
<ItemGroup>
<ProjectReference Include="../../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
<ProjectReference Include="../../../Concelier/__Libraries/StellaOps.Concelier.RawModels/StellaOps.Concelier.RawModels.csproj" />
<ProjectReference Include="../../../__Libraries/StellaOps.Ingestion.Telemetry/StellaOps.Ingestion.Telemetry.csproj" />
</ItemGroup>
</Project>

View File

@@ -62,7 +62,7 @@
## Observability & Forensics (Epic 15)
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| EXCITITOR-OBS-50-001 `Telemetry adoption` | TODO | Excititor Core Guild, Observability Guild | TELEMETRY-OBS-50-001, TELEMETRY-OBS-50-002 | Integrate telemetry core across VEX ingestion/linking, ensuring spans/logs capture tenant, product scope, upstream id, justification hash, and trace IDs. |
| EXCITITOR-OBS-50-001 `Telemetry adoption` | DONE (2025-11-07) | Excititor Core Guild, Observability Guild | TELEMETRY-OBS-50-001, TELEMETRY-OBS-50-002 | Integrate telemetry core across VEX ingestion/linking, ensuring spans/logs capture tenant, product scope, upstream id, justification hash, and trace IDs. |
| EXCITITOR-OBS-51-001 `Metrics & SLOs` | TODO | Excititor Core Guild, DevOps Guild | EXCITITOR-OBS-50-001, TELEMETRY-OBS-51-001 | Publish metrics for VEX ingest latency, scope resolution success, conflict rate, signature verification failures. Define SLOs (link latency P95 <30s) and configure burn-rate alerts. |
| EXCITITOR-OBS-52-001 `Timeline events` | TODO | Excititor Core Guild | EXCITITOR-OBS-50-001, TIMELINE-OBS-52-002 | Emit `timeline_event` entries for VEX ingest/linking/outcome changes with trace IDs, justification summaries, and evidence placeholders. |
| EXCITITOR-OBS-53-001 `Evidence snapshots` | TODO | Excititor Core Guild, Evidence Locker Guild | EXCITITOR-OBS-52-001, EVID-OBS-53-002 | Build evidence payloads for VEX statements (raw doc, normalization diff, precedence notes) and push to evidence locker with Merkle manifests. |

View File

@@ -68,6 +68,8 @@ public interface IVexConnectorStateRepository
ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null);
ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null);
}
public interface IVexConsensusHoldStore

View File

@@ -1,9 +1,10 @@
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
namespace StellaOps.Excititor.Storage.Mongo;
@@ -29,11 +30,11 @@ public sealed class MongoVexConnectorStateRepository : IVexConnectorStateReposit
return document?.ToRecord();
}
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
ArgumentNullException.ThrowIfNull(state);
var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests());
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
ArgumentNullException.ThrowIfNull(state);
var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests());
var filter = Builders<VexConnectorStateDocument>.Filter.Eq(x => x.ConnectorId, document.ConnectorId);
if (session is null)
{
@@ -41,10 +42,24 @@ public sealed class MongoVexConnectorStateRepository : IVexConnectorStateReposit
}
else
{
await _collection.ReplaceOneAsync(session, filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
}
}
}
await _collection.ReplaceOneAsync(session, filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
}
}
public async ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
var find = session is null
? _collection.Find(FilterDefinition<VexConnectorStateDocument>.Empty)
: _collection.Find(session, FilterDefinition<VexConnectorStateDocument>.Empty);
var documents = await find
.SortBy(x => x.ConnectorId)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents.ConvertAll(static document => document.ToRecord());
}
}
internal static class VexConnectorStateExtensions
{

View File

@@ -1,36 +1,34 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.Diagnostics;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.Core.Clusters;
using MongoDB.Driver.Core.Clusters;
using MongoDB.Driver.GridFS;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Aoc;
using RawContentMetadata = StellaOps.Concelier.RawModels.RawContent;
using RawLinkset = StellaOps.Concelier.RawModels.RawLinkset;
using RawReference = StellaOps.Concelier.RawModels.RawReference;
using RawSignatureMetadata = StellaOps.Concelier.RawModels.RawSignatureMetadata;
using RawSourceMetadata = StellaOps.Concelier.RawModels.RawSourceMetadata;
using RawUpstreamMetadata = StellaOps.Concelier.RawModels.RawUpstreamMetadata;
using RawVexDocument = StellaOps.Concelier.RawModels.VexRawDocument;
using VexStatementSummaryModel = StellaOps.Concelier.RawModels.VexStatementSummary;
using StellaOps.Ingestion.Telemetry;
namespace StellaOps.Excititor.Storage.Mongo;
public sealed class MongoVexRawStore : IVexRawStore
{
private readonly IMongoClient _client;
private readonly IMongoClient _client;
private readonly IMongoCollection<VexRawDocumentRecord> _collection;
private readonly GridFSBucket _bucket;
private readonly VexMongoStorageOptions _options;
private readonly IVexMongoSessionProvider _sessionProvider;
private readonly IVexRawWriteGuard _guard;
private readonly ILogger<MongoVexRawStore> _logger;
private readonly string _connectorVersion;
public MongoVexRawStore(
@@ -38,13 +36,15 @@ public sealed class MongoVexRawStore : IVexRawStore
IMongoDatabase database,
IOptions<VexMongoStorageOptions> options,
IVexMongoSessionProvider sessionProvider,
IVexRawWriteGuard guard)
IVexRawWriteGuard guard,
ILogger<MongoVexRawStore>? logger = null)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
ArgumentNullException.ThrowIfNull(database);
ArgumentNullException.ThrowIfNull(options);
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
_logger = logger ?? NullLogger<MongoVexRawStore>.Instance;
_options = options.Value;
Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true);
@@ -54,350 +54,285 @@ public sealed class MongoVexRawStore : IVexRawStore
_collection = database.GetCollection<VexRawDocumentRecord>(VexMongoCollectionNames.Raw);
_bucket = new GridFSBucket(database, new GridFSBucketOptions
{
BucketName = _options.RawBucketName,
ReadConcern = database.Settings.ReadConcern,
ReadPreference = database.Settings.ReadPreference,
WriteConcern = database.Settings.WriteConcern,
});
}
BucketName = _options.RawBucketName,
ReadConcern = database.Settings.ReadConcern,
ReadPreference = database.Settings.ReadPreference,
WriteConcern = database.Settings.WriteConcern,
});
}
public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
ArgumentNullException.ThrowIfNull(document);
var guardPayload = CreateRawModel(document);
_guard.EnsureValid(guardPayload);
var guardPayload = VexRawDocumentMapper.ToRawModel(document, _options.DefaultTenant);
var tenant = guardPayload.Tenant;
var sourceVendor = guardPayload.Source.Vendor;
var upstreamId = guardPayload.Upstream.UpstreamId;
var contentHash = guardPayload.Upstream.ContentHash;
using var logScope = _logger.BeginScope(new Dictionary<string, object?>(StringComparer.Ordinal)
{
["tenant"] = tenant,
["source.vendor"] = sourceVendor,
["upstream.upstreamId"] = upstreamId,
["contentHash"] = contentHash,
["providerId"] = document.ProviderId,
["digest"] = document.Digest,
});
var transformWatch = Stopwatch.StartNew();
using var transformActivity = IngestionTelemetry.StartTransformActivity(
tenant,
sourceVendor,
upstreamId,
contentHash,
document.Format.ToString(),
document.Content.Length);
try
{
_guard.EnsureValid(guardPayload);
transformActivity?.SetStatus(ActivityStatusCode.Ok);
}
catch (ExcititorAocGuardException ex)
{
transformActivity?.SetTag("violationCount", ex.Violations.IsDefaultOrEmpty ? 0 : ex.Violations.Length);
transformActivity?.SetTag("code", ex.PrimaryErrorCode);
transformActivity?.SetStatus(ActivityStatusCode.Error, ex.PrimaryErrorCode);
IngestionTelemetry.RecordViolation(tenant, sourceVendor, ex.PrimaryErrorCode);
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, IngestionTelemetry.ResultReject);
_logger.LogWarning(ex, "AOC guard rejected VEX document digest={Digest} provider={ProviderId}", document.Digest, document.ProviderId);
throw;
}
finally
{
if (transformWatch.IsRunning)
{
transformWatch.Stop();
}
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseTransform, transformWatch.Elapsed);
}
var threshold = _options.GridFsInlineThresholdBytes;
var useInline = threshold == 0 || document.Content.Length <= threshold;
string? newGridId = null;
string? oldGridIdToDelete = null;
var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
if (!useInline)
{
newGridId = await UploadToGridFsAsync(document, sessionHandle, cancellationToken).ConfigureAwait(false);
}
var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone
&& !sessionHandle.IsInTransaction;
var startedTransaction = false;
if (supportsTransactions)
{
try
{
sessionHandle.StartTransaction();
startedTransaction = true;
}
catch (NotSupportedException)
{
supportsTransactions = false;
}
}
try
{
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest);
var existing = await _collection
.Find(sessionHandle, filter)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline);
record.GridFsObjectId = useInline ? null : newGridId;
await _collection
.ReplaceOneAsync(
sessionHandle,
filter,
record,
new ReplaceOptions { IsUpsert = true },
cancellationToken)
.ConfigureAwait(false);
if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId))
{
if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal))
{
oldGridIdToDelete = oldGridId;
}
}
if (startedTransaction)
{
await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false);
}
}
catch
{
if (startedTransaction && sessionHandle.IsInTransaction)
{
await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false);
}
if (!useInline && !string.IsNullOrWhiteSpace(newGridId))
{
await DeleteFromGridFsAsync(newGridId, sessionHandle, cancellationToken).ConfigureAwait(false);
}
throw;
}
if (!string.IsNullOrWhiteSpace(oldGridIdToDelete))
{
await DeleteFromGridFsAsync(oldGridIdToDelete!, sessionHandle, cancellationToken).ConfigureAwait(false);
}
}
public async ValueTask<VexRawDocument?> FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
if (string.IsNullOrWhiteSpace(digest))
{
throw new ArgumentException("Digest must be provided.", nameof(digest));
}
var trimmed = digest.Trim();
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, trimmed);
var record = session is null
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (record is null)
{
return null;
}
if (!string.IsNullOrWhiteSpace(record.GridFsObjectId))
{
var handle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, handle, cancellationToken).ConfigureAwait(false);
return record.ToDomain(new ReadOnlyMemory<byte>(bytes));
}
return record.ToDomain();
}
private async Task<string?> UploadToGridFsAsync(VexRawDocument document, IClientSessionHandle? session, CancellationToken cancellationToken)
{
using var stream = new MemoryStream(document.Content.ToArray(), writable: false);
var metadata = new BsonDocument
{
{ "providerId", document.ProviderId },
{ "format", document.Format.ToString().ToLowerInvariant() },
{ "sourceUri", document.SourceUri.ToString() },
{ "retrievedAt", document.RetrievedAt.UtcDateTime },
};
var options = new GridFSUploadOptions { Metadata = metadata };
var objectId = await _bucket
.UploadFromStreamAsync(document.Digest, stream, options, cancellationToken)
.ConfigureAwait(false);
return objectId.ToString();
}
private async Task DeleteFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
{
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
{
return;
}
try
{
await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false);
}
catch (GridFSFileNotFoundException)
{
// file already removed by TTL or manual cleanup
}
}
private async Task<byte[]> DownloadFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
{
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
{
return Array.Empty<byte>();
}
return await _bucket.DownloadAsBytesAsync(objectId, null, cancellationToken).ConfigureAwait(false);
}
async ValueTask IVexRawDocumentSink.StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
=> await StoreAsync(document, cancellationToken, session: null).ConfigureAwait(false);
var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
private RawVexDocument CreateRawModel(VexRawDocument document)
{
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
var tenant = _options.DefaultTenant;
if (!useInline)
{
newGridId = await UploadToGridFsAsync(document, sessionHandle, cancellationToken).ConfigureAwait(false);
}
var source = CreateSourceMetadata(document, metadata);
var content = CreateContent(document, metadata);
var upstream = CreateUpstreamMetadata(document, metadata);
var linkset = CreateLinkset();
var statements = ImmutableArray<VexStatementSummaryModel>.Empty;
var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone
&& !sessionHandle.IsInTransaction;
return new RawVexDocument(
var startedTransaction = false;
if (supportsTransactions)
{
try
{
sessionHandle.StartTransaction();
startedTransaction = true;
}
catch (NotSupportedException)
{
supportsTransactions = false;
}
}
var fetchWatch = Stopwatch.StartNew();
using var fetchActivity = IngestionTelemetry.StartFetchActivity(
tenant,
source,
upstream,
content,
linkset,
statements);
}
private RawSourceMetadata CreateSourceMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
var vendor = TryMetadata(metadata, "source.vendor", "connector.vendor") ?? ExtractVendor(document.ProviderId);
var connector = TryMetadata(metadata, "source.connector") ?? document.ProviderId;
var version = TryMetadata(metadata, "source.connector_version", "connector.version") ?? _connectorVersion;
var stream = TryMetadata(metadata, "source.stream", "connector.stream") ?? document.Format.ToString().ToLowerInvariant();
return new RawSourceMetadata(
vendor,
connector,
version,
stream);
}
private RawUpstreamMetadata CreateUpstreamMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
var upstreamId = TryMetadata(
metadata,
"upstream.id",
"aoc.upstream_id",
"vulnerability.id",
"advisory.id",
"msrc.vulnerabilityId",
"msrc.advisoryId",
"oracle.csaf.entryId",
"ubuntu.advisoryId",
"cisco.csaf.documentId",
"rancher.vex.id") ?? document.SourceUri.ToString();
var documentVersion = TryMetadata(
metadata,
"upstream.version",
"aoc.document_version",
"msrc.lastModified",
"msrc.releaseDate",
"oracle.csaf.revision",
"ubuntu.version",
"ubuntu.lastModified",
"cisco.csaf.revision") ?? document.RetrievedAt.ToString("O");
var signature = CreateSignatureMetadata(metadata);
var provenance = metadata;
return new RawUpstreamMetadata(
sourceVendor,
upstreamId,
documentVersion,
document.RetrievedAt,
document.Digest,
signature,
provenance);
}
contentHash,
document.SourceUri.ToString());
fetchActivity?.SetTag("providerId", document.ProviderId);
fetchActivity?.SetTag("format", document.Format.ToString().ToLowerInvariant());
private static RawSignatureMetadata CreateSignatureMetadata(ImmutableDictionary<string, string> metadata)
{
if (!TryBool(metadata, out var present, "signature.present", "aoc.signature.present"))
VexRawDocumentRecord? existing;
try
{
return new RawSignatureMetadata(false);
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest);
existing = await _collection
.Find(sessionHandle, filter)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
fetchActivity?.SetTag("result", existing is null ? "miss" : "hit");
fetchActivity?.SetStatus(ActivityStatusCode.Ok);
}
catch
{
fetchActivity?.SetStatus(ActivityStatusCode.Error, "lookup-failed");
throw;
}
finally
{
if (fetchWatch.IsRunning)
{
fetchWatch.Stop();
}
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseFetch, fetchWatch.Elapsed);
}
if (!present)
var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline);
record.GridFsObjectId = useInline ? null : newGridId;
var writeWatch = Stopwatch.StartNew();
using var writeActivity = IngestionTelemetry.StartWriteActivity(
tenant,
sourceVendor,
upstreamId,
contentHash,
VexMongoCollectionNames.Raw);
string? writeResult = null;
try
{
return new RawSignatureMetadata(false);
await _collection
.ReplaceOneAsync(
sessionHandle,
Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest),
record,
new ReplaceOptions { IsUpsert = true },
cancellationToken)
.ConfigureAwait(false);
writeResult = existing is null ? IngestionTelemetry.ResultOk : IngestionTelemetry.ResultNoop;
writeActivity?.SetTag("result", writeResult);
if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId))
{
if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal))
{
oldGridIdToDelete = oldGridId;
}
}
if (startedTransaction)
{
await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false);
}
writeActivity?.SetStatus(ActivityStatusCode.Ok);
}
catch
{
if (startedTransaction && sessionHandle.IsInTransaction)
{
await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false);
}
if (!useInline && !string.IsNullOrWhiteSpace(newGridId))
{
await DeleteFromGridFsAsync(newGridId, sessionHandle, cancellationToken).ConfigureAwait(false);
}
writeActivity?.SetStatus(ActivityStatusCode.Error, "write-failed");
throw;
}
finally
{
if (writeWatch.IsRunning)
{
writeWatch.Stop();
}
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseWrite, writeWatch.Elapsed);
if (!string.IsNullOrEmpty(writeResult))
{
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, writeResult);
}
}
var format = TryMetadata(metadata, "signature.format", "aoc.signature.format");
var keyId = TryMetadata(metadata, "signature.key_id", "signature.keyId", "aoc.signature.key_id");
var signature = TryMetadata(metadata, "signature.sig", "signature.signature", "aoc.signature.sig");
var digest = TryMetadata(metadata, "signature.digest", "aoc.signature.digest");
var certificate = TryMetadata(metadata, "signature.certificate", "aoc.signature.certificate");
return new RawSignatureMetadata(true, format, keyId, signature, certificate, digest);
if (!string.IsNullOrWhiteSpace(oldGridIdToDelete))
{
await DeleteFromGridFsAsync(oldGridIdToDelete!, sessionHandle, cancellationToken).ConfigureAwait(false);
}
}
private RawContentMetadata CreateContent(VexRawDocument document, ImmutableDictionary<string, string> metadata)
public async ValueTask<VexRawDocument?> FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
if (document.Content.IsEmpty)
if (string.IsNullOrWhiteSpace(digest))
{
throw new InvalidOperationException("Raw VEX document content cannot be empty when enforcing AOC guard.");
throw new ArgumentException("Digest must be provided.", nameof(digest));
}
var trimmed = digest.Trim();
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, trimmed);
var record = session is null
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (record is null)
{
return null;
}
if (!string.IsNullOrWhiteSpace(record.GridFsObjectId))
{
var handle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, handle, cancellationToken).ConfigureAwait(false);
return record.ToDomain(new ReadOnlyMemory<byte>(bytes));
}
return record.ToDomain();
}
private async Task<string?> UploadToGridFsAsync(VexRawDocument document, IClientSessionHandle? session, CancellationToken cancellationToken)
{
using var stream = new MemoryStream(document.Content.ToArray(), writable: false);
var metadata = new BsonDocument
{
{ "providerId", document.ProviderId },
{ "format", document.Format.ToString().ToLowerInvariant() },
{ "sourceUri", document.SourceUri.ToString() },
{ "retrievedAt", document.RetrievedAt.UtcDateTime },
};
var options = new GridFSUploadOptions { Metadata = metadata };
var objectId = await _bucket
.UploadFromStreamAsync(document.Digest, stream, options, cancellationToken)
.ConfigureAwait(false);
return objectId.ToString();
}
private async Task DeleteFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
{
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
{
return;
}
try
{
using var payload = JsonDocument.Parse(document.Content.ToArray());
var raw = payload.RootElement.Clone();
var specVersion = TryMetadata(metadata, "content.spec_version", "csaf.version", "openvex.version");
var encoding = TryMetadata(metadata, "content.encoding");
return new RawContentMetadata(
document.Format.ToString(),
specVersion,
raw,
encoding);
await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false);
}
catch (JsonException ex)
catch (GridFSFileNotFoundException)
{
throw new InvalidOperationException("Raw VEX document payload must be valid JSON for AOC guard enforcement.", ex);
// file already removed by TTL or manual cleanup
}
}
private static RawLinkset CreateLinkset()
=> new()
{
Aliases = ImmutableArray<string>.Empty,
PackageUrls = ImmutableArray<string>.Empty,
Cpes = ImmutableArray<string>.Empty,
References = ImmutableArray<RawReference>.Empty,
ReconciledFrom = ImmutableArray<string>.Empty,
Notes = ImmutableDictionary<string, string>.Empty,
};
private static string? TryMetadata(ImmutableDictionary<string, string> metadata, params string[] keys)
private async Task<byte[]> DownloadFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
{
foreach (var key in keys)
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
{
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value;
}
return Array.Empty<byte>();
}
return null;
return await _bucket.DownloadAsBytesAsync(objectId, null, cancellationToken).ConfigureAwait(false);
}
private static bool TryBool(ImmutableDictionary<string, string> metadata, out bool value, params string[] keys)
{
foreach (var key in keys)
{
if (metadata.TryGetValue(key, out var text) && bool.TryParse(text, out value))
{
return true;
}
}
value = default;
return false;
}
private static string ExtractVendor(string providerId)
{
if (string.IsNullOrWhiteSpace(providerId))
{
return "unknown";
}
var trimmed = providerId.Trim();
var separatorIndex = trimmed.LastIndexOfAny(new[] { ':', '.' });
if (separatorIndex >= 0 && separatorIndex < trimmed.Length - 1)
{
return trimmed[(separatorIndex + 1)..];
}
return trimmed;
}
async ValueTask IVexRawDocumentSink.StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
=> await StoreAsync(document, cancellationToken, session: null).ConfigureAwait(false);
}

View File

@@ -12,7 +12,8 @@
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
</ItemGroup>
</Project>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Ingestion.Telemetry\StellaOps.Ingestion.Telemetry.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,194 @@
using System;
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Excititor.Core;
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
using RawSourceMetadata = StellaOps.Concelier.RawModels.RawSourceMetadata;
using RawUpstreamMetadata = StellaOps.Concelier.RawModels.RawUpstreamMetadata;
using RawContentMetadata = StellaOps.Concelier.RawModels.RawContent;
using RawSignatureMetadata = StellaOps.Concelier.RawModels.RawSignatureMetadata;
using RawLinkset = StellaOps.Concelier.RawModels.RawLinkset;
using RawReference = StellaOps.Concelier.RawModels.RawReference;
using VexStatementSummaryModel = StellaOps.Concelier.RawModels.VexStatementSummary;
namespace StellaOps.Excititor.Storage.Mongo;
/// <summary>
/// Converts Excititor domain VEX documents into Aggregation-Only Contract raw payloads.
/// </summary>
public static class VexRawDocumentMapper
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
public static RawVexDocumentModel ToRawModel(VexRawDocument document, string defaultTenant)
{
ArgumentNullException.ThrowIfNull(document);
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
var tenant = ResolveTenant(metadata, defaultTenant);
var source = CreateSourceMetadata(document, metadata);
var upstream = CreateUpstreamMetadata(document, metadata);
var content = CreateContent(document, metadata);
var linkset = CreateLinkset();
ImmutableArray<VexStatementSummaryModel>? statements = null;
return new RawVexDocumentModel(tenant, source, upstream, content, linkset, statements);
}
private static string ResolveTenant(ImmutableDictionary<string, string> metadata, string fallback)
{
var tenant = TryMetadata(metadata, "tenant", "aoc.tenant");
if (string.IsNullOrWhiteSpace(tenant))
{
return (fallback ?? "tenant-default").Trim().ToLowerInvariant();
}
return tenant.Trim().ToLowerInvariant();
}
private static RawSourceMetadata CreateSourceMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
var vendor = TryMetadata(metadata, "source.vendor", "connector.vendor") ?? ExtractVendor(document.ProviderId);
var connector = TryMetadata(metadata, "source.connector") ?? document.ProviderId;
var version = TryMetadata(metadata, "source.connector_version", "connector.version") ?? GetAssemblyVersion();
var stream = TryMetadata(metadata, "source.stream", "connector.stream") ?? document.Format.ToString().ToLowerInvariant();
return new RawSourceMetadata(vendor, connector, version, stream);
}
private static string GetAssemblyVersion()
=> typeof(VexRawDocumentMapper).Assembly.GetName().Version?.ToString() ?? "0.0.0";
private static RawUpstreamMetadata CreateUpstreamMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
var upstreamId = TryMetadata(
metadata,
"upstream.id",
"aoc.upstream_id",
"vulnerability.id",
"advisory.id",
"msrc.vulnerabilityId",
"msrc.advisoryId",
"oracle.csaf.entryId",
"ubuntu.advisoryId",
"cisco.csaf.documentId",
"rancher.vex.id") ?? document.SourceUri.ToString();
var documentVersion = TryMetadata(
metadata,
"upstream.version",
"aoc.document_version",
"msrc.lastModified",
"msrc.releaseDate",
"oracle.csaf.revision",
"ubuntu.version",
"ubuntu.lastModified",
"cisco.csaf.revision") ?? document.RetrievedAt.ToString("O");
var signature = CreateSignatureMetadata(metadata);
return new RawUpstreamMetadata(
upstreamId,
documentVersion,
document.RetrievedAt,
document.Digest,
signature,
metadata);
}
private static RawSignatureMetadata CreateSignatureMetadata(ImmutableDictionary<string, string> metadata)
{
if (!TryBool(metadata, out var present, "signature.present", "aoc.signature.present"))
{
return new RawSignatureMetadata(false);
}
if (!present)
{
return new RawSignatureMetadata(false);
}
var format = TryMetadata(metadata, "signature.format", "aoc.signature.format");
var keyId = TryMetadata(metadata, "signature.key_id", "signature.keyId", "aoc.signature.key_id");
var signature = TryMetadata(metadata, "signature.sig", "signature.signature", "aoc.signature.sig");
var digest = TryMetadata(metadata, "signature.digest", "aoc.signature.digest");
var certificate = TryMetadata(metadata, "signature.certificate", "aoc.signature.certificate");
return new RawSignatureMetadata(true, format, keyId, signature, certificate, digest);
}
private static RawContentMetadata CreateContent(VexRawDocument document, ImmutableDictionary<string, string> metadata)
{
if (document.Content.IsEmpty)
{
throw new InvalidOperationException("Raw VEX document content cannot be empty when enforcing AOC guard.");
}
try
{
using var payload = JsonDocument.Parse(document.Content.ToArray());
var raw = payload.RootElement.Clone();
var specVersion = TryMetadata(metadata, "content.spec_version", "csaf.version", "openvex.version");
var encoding = TryMetadata(metadata, "content.encoding");
return new RawContentMetadata(
document.Format.ToString(),
specVersion,
raw,
encoding);
}
catch (JsonException ex)
{
throw new InvalidOperationException("Raw VEX document payload must be valid JSON for AOC guard enforcement.", ex);
}
}
private static RawLinkset CreateLinkset()
=> new()
{
Aliases = ImmutableArray<string>.Empty,
PackageUrls = ImmutableArray<string>.Empty,
Cpes = ImmutableArray<string>.Empty,
References = ImmutableArray<RawReference>.Empty,
ReconciledFrom = ImmutableArray<string>.Empty,
Notes = ImmutableDictionary<string, string>.Empty,
};
private static string? TryMetadata(ImmutableDictionary<string, string> metadata, params string[] keys)
{
foreach (var key in keys)
{
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
{
return value;
}
}
return null;
}
private static bool TryBool(ImmutableDictionary<string, string> metadata, out bool value, params string[] keys)
{
foreach (var key in keys)
{
if (metadata.TryGetValue(key, out var text) && bool.TryParse(text, out value))
{
return true;
}
}
value = default;
return false;
}
private static string ExtractVendor(string providerId)
{
if (string.IsNullOrWhiteSpace(providerId))
{
return "unknown";
}
var trimmed = providerId.Trim();
var separatorIndex = trimmed.LastIndexOfAny(new[] { ':', '.' });
if (separatorIndex >= 0 && separatorIndex < trimmed.Length - 1)
{
return trimmed[(separatorIndex + 1)..];
}
return trimmed;
}
}