Add tests and implement timeline ingestion options with NATS and Redis subscribers
- Introduced `BinaryReachabilityLifterTests` to validate binary lifting functionality. - Created `PackRunWorkerOptions` for configuring worker paths and execution persistence. - Added `TimelineIngestionOptions` for configuring NATS and Redis ingestion transports. - Implemented `NatsTimelineEventSubscriber` for subscribing to NATS events. - Developed `RedisTimelineEventSubscriber` for reading from Redis Streams. - Added `TimelineEnvelopeParser` to normalize incoming event envelopes. - Created unit tests for `TimelineEnvelopeParser` to ensure correct field mapping. - Implemented `TimelineAuthorizationAuditSink` for logging authorization outcomes.
This commit is contained in:
StellaOps Bot
@@ -1,37 +0,0 @@
|
||||
# Excititor Storage (Mongo) Charter
|
||||
|
||||
## Mission
|
||||
Provide Mongo-backed persistence for Excititor ingestion, linksets, and observations with deterministic schemas, indexes, and migrations; keep aggregation-only semantics intact.
|
||||
|
||||
## Scope
|
||||
- Working directory: `src/Excititor/__Libraries/StellaOps.Excititor.Storage.Mongo`
|
||||
- Collections, indexes, migrations, repository abstractions, and data access helpers shared by WebService/Worker/Core.
|
||||
|
||||
## Required Reading
|
||||
- `docs/modules/excititor/architecture.md`
|
||||
- `docs/modules/excititor/vex_observations.md`
|
||||
- `docs/ingestion/aggregation-only-contract.md`
|
||||
- `docs/modules/excititor/implementation_plan.md`
|
||||
|
||||
## Roles
|
||||
- Backend/storage engineer (.NET 10, MongoDB driver ≥3.0).
|
||||
- QA automation (repository + migration tests).
|
||||
|
||||
## Working Agreements
|
||||
1. Maintain deterministic migrations; record new indexes and shapes in sprint `Execution Log` and module docs if added.
|
||||
2. Enforce tenant scope in all queries; include partition keys in indexes where applicable.
|
||||
3. No consensus/weighting logic; store raw facts, provenance, and precedence pointers only.
|
||||
4. Offline-first; no runtime external calls.
|
||||
|
||||
## Testing & Determinism
|
||||
- Use Mongo test fixtures/in-memory harness with seeded data; assert index presence and sort stability.
|
||||
- Keep timestamps UTC ISO-8601 and ordering explicit (e.g., vendor, upstreamId, version, createdUtc).
|
||||
- Avoid nondeterministic ObjectId/GUID usage in tests; seed values.
|
||||
|
||||
## Boundaries
|
||||
- Do not embed Policy Engine or Cartographer schemas; consume published contracts.
|
||||
- Config via DI/appsettings; no hard-coded connection strings.
|
||||
|
||||
## Ready-to-Start Checklist
|
||||
- Required docs reviewed.
|
||||
- Test fixture database prepared; migrations scripted and reversible where possible.
|
||||
@@ -1,173 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public interface IAirgapImportStore
|
||||
{
|
||||
Task SaveAsync(AirgapImportRecord record, CancellationToken cancellationToken);
|
||||
|
||||
Task<AirgapImportRecord?> FindByBundleIdAsync(
|
||||
string tenantId,
|
||||
string bundleId,
|
||||
string? mirrorGeneration,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<IReadOnlyList<AirgapImportRecord>> ListAsync(
|
||||
string tenantId,
|
||||
string? publisherFilter,
|
||||
DateTimeOffset? importedAfter,
|
||||
int limit,
|
||||
int offset,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<int> CountAsync(
|
||||
string tenantId,
|
||||
string? publisherFilter,
|
||||
DateTimeOffset? importedAfter,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class DuplicateAirgapImportException : Exception
|
||||
{
|
||||
public string BundleId { get; }
|
||||
public string MirrorGeneration { get; }
|
||||
|
||||
public DuplicateAirgapImportException(string bundleId, string mirrorGeneration, Exception inner)
|
||||
: base($"Airgap import already exists for bundle '{bundleId}' generation '{mirrorGeneration}'.", inner)
|
||||
{
|
||||
BundleId = bundleId;
|
||||
MirrorGeneration = mirrorGeneration;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MongoAirgapImportStore : IAirgapImportStore
|
||||
{
|
||||
private readonly IMongoCollection<AirgapImportRecord> _collection;
|
||||
|
||||
public MongoAirgapImportStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<AirgapImportRecord>(VexMongoCollectionNames.AirgapImports);
|
||||
|
||||
// Enforce idempotency on (bundleId, generation) via Id uniqueness and explicit index.
|
||||
var idIndex = Builders<AirgapImportRecord>.IndexKeys.Ascending(x => x.Id);
|
||||
var bundleIndex = Builders<AirgapImportRecord>.IndexKeys
|
||||
.Ascending(x => x.BundleId)
|
||||
.Ascending(x => x.MirrorGeneration);
|
||||
|
||||
_collection.Indexes.CreateMany(new[]
|
||||
{
|
||||
new CreateIndexModel<AirgapImportRecord>(idIndex, new CreateIndexOptions { Unique = true, Name = "airgap_import_id_unique" }),
|
||||
new CreateIndexModel<AirgapImportRecord>(bundleIndex, new CreateIndexOptions { Unique = true, Name = "airgap_bundle_generation_unique" })
|
||||
});
|
||||
}
|
||||
|
||||
public Task SaveAsync(AirgapImportRecord record, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(record);
|
||||
try
|
||||
{
|
||||
return _collection.InsertOneAsync(record, cancellationToken: cancellationToken);
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
throw new DuplicateAirgapImportException(record.BundleId, record.MirrorGeneration, ex);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<AirgapImportRecord?> FindByBundleIdAsync(
|
||||
string tenantId,
|
||||
string bundleId,
|
||||
string? mirrorGeneration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(bundleId);
|
||||
|
||||
var filter = Builders<AirgapImportRecord>.Filter.And(
|
||||
Builders<AirgapImportRecord>.Filter.Eq(x => x.TenantId, tenantId),
|
||||
Builders<AirgapImportRecord>.Filter.Eq(x => x.BundleId, bundleId));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(mirrorGeneration))
|
||||
{
|
||||
filter = Builders<AirgapImportRecord>.Filter.And(
|
||||
filter,
|
||||
Builders<AirgapImportRecord>.Filter.Eq(x => x.MirrorGeneration, mirrorGeneration));
|
||||
}
|
||||
|
||||
var sort = Builders<AirgapImportRecord>.Sort.Descending(x => x.MirrorGeneration);
|
||||
|
||||
return await _collection
|
||||
.Find(filter)
|
||||
.Sort(sort)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AirgapImportRecord>> ListAsync(
|
||||
string tenantId,
|
||||
string? publisherFilter,
|
||||
DateTimeOffset? importedAfter,
|
||||
int limit,
|
||||
int offset,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenantId);
|
||||
|
||||
var filter = BuildListFilter(tenantId, publisherFilter, importedAfter);
|
||||
var sort = Builders<AirgapImportRecord>.Sort.Descending(x => x.ImportedAt);
|
||||
|
||||
return await _collection
|
||||
.Find(filter)
|
||||
.Sort(sort)
|
||||
.Skip(offset)
|
||||
.Limit(Math.Clamp(limit, 1, 1000))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<int> CountAsync(
|
||||
string tenantId,
|
||||
string? publisherFilter,
|
||||
DateTimeOffset? importedAfter,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(tenantId);
|
||||
|
||||
var filter = BuildListFilter(tenantId, publisherFilter, importedAfter);
|
||||
|
||||
var count = await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return (int)Math.Min(count, int.MaxValue);
|
||||
}
|
||||
|
||||
private static FilterDefinition<AirgapImportRecord> BuildListFilter(
|
||||
string tenantId,
|
||||
string? publisherFilter,
|
||||
DateTimeOffset? importedAfter)
|
||||
{
|
||||
var filters = new List<FilterDefinition<AirgapImportRecord>>
|
||||
{
|
||||
Builders<AirgapImportRecord>.Filter.Eq(x => x.TenantId, tenantId)
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(publisherFilter))
|
||||
{
|
||||
filters.Add(Builders<AirgapImportRecord>.Filter.Eq(x => x.Publisher, publisherFilter));
|
||||
}
|
||||
|
||||
if (importedAfter is { } after)
|
||||
{
|
||||
filters.Add(Builders<AirgapImportRecord>.Filter.Gte(x => x.ImportedAt, after));
|
||||
}
|
||||
|
||||
return Builders<AirgapImportRecord>.Filter.And(filters);
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
using System;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
[BsonIgnoreExtraElements]
|
||||
public sealed class AirgapTimelineEntry
|
||||
{
|
||||
public string EventType { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
|
||||
|
||||
public string TenantId { get; set; } = "default";
|
||||
|
||||
public string BundleId { get; set; } = string.Empty;
|
||||
|
||||
public string MirrorGeneration { get; set; } = string.Empty;
|
||||
|
||||
public int? StalenessSeconds { get; set; }
|
||||
= null;
|
||||
|
||||
public string? ErrorCode { get; set; }
|
||||
= null;
|
||||
|
||||
public string? Message { get; set; }
|
||||
= null;
|
||||
|
||||
public string? Remediation { get; set; }
|
||||
= null;
|
||||
|
||||
public string? Actor { get; set; }
|
||||
= null;
|
||||
|
||||
public string? Scopes { get; set; }
|
||||
= null;
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public interface IVexAttestationLinkStore
|
||||
{
|
||||
ValueTask UpsertAsync(VexAttestationPayload payload, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<VexAttestationPayload?> FindAsync(string attestationId, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public interface IVexExportStore
|
||||
{
|
||||
ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public interface IVexRawStore : IVexRawDocumentSink
|
||||
{
|
||||
ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<VexRawDocument?> FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public interface IVexProviderStore
|
||||
{
|
||||
ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexConsensusStore
|
||||
{
|
||||
ValueTask<VexConsensus?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexConsensus>> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
IAsyncEnumerable<VexConsensus> FindCalculatedBeforeAsync(DateTimeOffset cutoff, int batchSize, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public interface IVexClaimStore
|
||||
{
|
||||
ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves all claims for a specific vulnerability ID (EXCITITOR-VULN-29-002).
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public sealed record VexConnectorState(
|
||||
string ConnectorId,
|
||||
DateTimeOffset? LastUpdated,
|
||||
ImmutableArray<string> DocumentDigests,
|
||||
ImmutableDictionary<string, string> ResumeTokens,
|
||||
DateTimeOffset? LastSuccessAt,
|
||||
int FailureCount,
|
||||
DateTimeOffset? NextEligibleRun,
|
||||
string? LastFailureReason,
|
||||
DateTimeOffset? LastHeartbeatAt = null,
|
||||
string? LastHeartbeatStatus = null,
|
||||
string? LastArtifactHash = null,
|
||||
string? LastArtifactKind = null,
|
||||
string? LastCheckpoint = null)
|
||||
{
|
||||
public VexConnectorState(
|
||||
string connectorId,
|
||||
DateTimeOffset? lastUpdated,
|
||||
ImmutableArray<string> documentDigests)
|
||||
: this(
|
||||
connectorId,
|
||||
lastUpdated,
|
||||
documentDigests,
|
||||
ImmutableDictionary<string, string>.Empty,
|
||||
LastSuccessAt: null,
|
||||
FailureCount: 0,
|
||||
NextEligibleRun: null,
|
||||
LastFailureReason: null,
|
||||
LastHeartbeatAt: null,
|
||||
LastHeartbeatStatus: null,
|
||||
LastArtifactHash: null,
|
||||
LastArtifactKind: null,
|
||||
LastCheckpoint: null)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public interface IVexConnectorStateRepository
|
||||
{
|
||||
ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexConsensusHoldStore
|
||||
{
|
||||
ValueTask<VexConsensusHold?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConsensusHold hold, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask RemoveAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
IAsyncEnumerable<VexConsensusHold> FindEligibleAsync(DateTimeOffset asOf, int batchSize, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexCacheIndex
|
||||
{
|
||||
ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexCacheMaintenance
|
||||
{
|
||||
ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
internal interface IVexMongoMigration
|
||||
{
|
||||
string Id { get; }
|
||||
|
||||
ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
internal sealed class VexConsensusHoldMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251021-consensus-holds";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<VexConsensusHoldRecord>(VexMongoCollectionNames.ConsensusHolds);
|
||||
|
||||
var eligibleIndex = Builders<VexConsensusHoldRecord>.IndexKeys
|
||||
.Ascending(x => x.EligibleAt);
|
||||
|
||||
var keyIndex = Builders<VexConsensusHoldRecord>.IndexKeys
|
||||
.Ascending(x => x.VulnerabilityId)
|
||||
.Ascending(x => x.ProductKey);
|
||||
|
||||
await Task.WhenAll(
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexConsensusHoldRecord>(eligibleIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexConsensusHoldRecord>(keyIndex), cancellationToken: cancellationToken))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
internal sealed class VexConsensusSignalsMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251019-consensus-signals-statements";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
await EnsureConsensusIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureStatementIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static Task EnsureConsensusIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexConsensusRecord>(VexMongoCollectionNames.Consensus);
|
||||
var revisionGeneratedIndex = Builders<VexConsensusRecord>.IndexKeys
|
||||
.Ascending(x => x.PolicyRevisionId)
|
||||
.Descending(x => x.CalculatedAt);
|
||||
|
||||
return collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexConsensusRecord>(revisionGeneratedIndex),
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
private static Task EnsureStatementIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexStatementRecord>(VexMongoCollectionNames.Statements);
|
||||
|
||||
var vulnProductInsertedIndex = Builders<VexStatementRecord>.IndexKeys
|
||||
.Ascending(x => x.VulnerabilityId)
|
||||
.Ascending(x => x.Product.Key)
|
||||
.Descending(x => x.InsertedAt);
|
||||
|
||||
var providerInsertedIndex = Builders<VexStatementRecord>.IndexKeys
|
||||
.Ascending(x => x.ProviderId)
|
||||
.Descending(x => x.InsertedAt);
|
||||
|
||||
var digestIndex = Builders<VexStatementRecord>.IndexKeys
|
||||
.Ascending(x => x.Document.Digest);
|
||||
|
||||
return Task.WhenAll(
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexStatementRecord>(vulnProductInsertedIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexStatementRecord>(providerInsertedIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexStatementRecord>(digestIndex), cancellationToken: cancellationToken));
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
internal sealed class VexInitialIndexMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251016-initial-indexes";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
await EnsureRawIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureProviderIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureConsensusIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureExportIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureCacheIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static Task EnsureRawIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexRawDocumentRecord>(VexMongoCollectionNames.Raw);
|
||||
var providerFormatIndex = Builders<VexRawDocumentRecord>.IndexKeys
|
||||
.Ascending(x => x.ProviderId)
|
||||
.Ascending(x => x.Format)
|
||||
.Ascending(x => x.RetrievedAt);
|
||||
return collection.Indexes.CreateOneAsync(new CreateIndexModel<VexRawDocumentRecord>(providerFormatIndex), cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
private static Task EnsureProviderIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexProviderRecord>(VexMongoCollectionNames.Providers);
|
||||
var kindIndex = Builders<VexProviderRecord>.IndexKeys.Ascending(x => x.Kind);
|
||||
return collection.Indexes.CreateOneAsync(new CreateIndexModel<VexProviderRecord>(kindIndex), cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
private static Task EnsureConsensusIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexConsensusRecord>(VexMongoCollectionNames.Consensus);
|
||||
var vulnProductIndex = Builders<VexConsensusRecord>.IndexKeys
|
||||
.Ascending(x => x.VulnerabilityId)
|
||||
.Ascending(x => x.Product.Key);
|
||||
var policyIndex = Builders<VexConsensusRecord>.IndexKeys
|
||||
.Ascending(x => x.PolicyRevisionId)
|
||||
.Ascending(x => x.PolicyDigest);
|
||||
|
||||
return Task.WhenAll(
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexConsensusRecord>(vulnProductIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexConsensusRecord>(policyIndex), cancellationToken: cancellationToken));
|
||||
}
|
||||
|
||||
private static Task EnsureExportIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexExportManifestRecord>(VexMongoCollectionNames.Exports);
|
||||
var signatureIndex = Builders<VexExportManifestRecord>.IndexKeys
|
||||
.Ascending(x => x.QuerySignature)
|
||||
.Ascending(x => x.Format);
|
||||
return collection.Indexes.CreateOneAsync(new CreateIndexModel<VexExportManifestRecord>(signatureIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
private static Task EnsureCacheIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache);
|
||||
var signatureIndex = Builders<VexCacheEntryRecord>.IndexKeys
|
||||
.Ascending(x => x.QuerySignature)
|
||||
.Ascending(x => x.Format);
|
||||
var expirationIndex = Builders<VexCacheEntryRecord>.IndexKeys.Ascending(x => x.ExpiresAt);
|
||||
|
||||
return Task.WhenAll(
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexCacheEntryRecord>(signatureIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexCacheEntryRecord>(expirationIndex, new CreateIndexOptions { ExpireAfter = TimeSpan.FromSeconds(0) }), cancellationToken: cancellationToken));
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
using System;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
internal sealed class VexMigrationRecord
|
||||
{
|
||||
public VexMigrationRecord(string id, DateTimeOffset executedAt)
|
||||
{
|
||||
Id = string.IsNullOrWhiteSpace(id) ? throw new ArgumentException("Migration id must be provided.", nameof(id)) : id.Trim();
|
||||
ExecutedAt = executedAt;
|
||||
}
|
||||
|
||||
[BsonId]
|
||||
public string Id { get; }
|
||||
|
||||
public DateTimeOffset ExecutedAt { get; }
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
internal sealed class VexMongoMigrationHostedService : IHostedService
|
||||
{
|
||||
private readonly VexMongoMigrationRunner _runner;
|
||||
|
||||
public VexMongoMigrationHostedService(VexMongoMigrationRunner runner)
|
||||
{
|
||||
_runner = runner ?? throw new ArgumentNullException(nameof(runner));
|
||||
}
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await _runner.RunAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
internal sealed class VexMongoMigrationRunner
|
||||
{
|
||||
private readonly IMongoDatabase _database;
|
||||
private readonly IReadOnlyList<IVexMongoMigration> _migrations;
|
||||
private readonly ILogger<VexMongoMigrationRunner> _logger;
|
||||
|
||||
public VexMongoMigrationRunner(
|
||||
IMongoDatabase database,
|
||||
IEnumerable<IVexMongoMigration> migrations,
|
||||
ILogger<VexMongoMigrationRunner> logger)
|
||||
{
|
||||
_database = database ?? throw new ArgumentNullException(nameof(database));
|
||||
_migrations = (migrations ?? throw new ArgumentNullException(nameof(migrations)))
|
||||
.OrderBy(migration => migration.Id, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async ValueTask RunAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_migrations.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var migrationsCollection = _database.GetCollection<VexMigrationRecord>(VexMongoCollectionNames.Migrations);
|
||||
await EnsureMigrationsIndexAsync(migrationsCollection, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var applied = await LoadAppliedMigrationsAsync(migrationsCollection, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var migration in _migrations)
|
||||
{
|
||||
if (applied.Contains(migration.Id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Applying Excititor Mongo migration {MigrationId}", migration.Id);
|
||||
await migration.ExecuteAsync(_database, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var record = new VexMigrationRecord(migration.Id, DateTimeOffset.UtcNow);
|
||||
await migrationsCollection.InsertOneAsync(record, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation("Completed Excititor Mongo migration {MigrationId}", migration.Id);
|
||||
}
|
||||
}
|
||||
|
||||
private static ValueTask EnsureMigrationsIndexAsync(
|
||||
IMongoCollection<VexMigrationRecord> collection,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// default _id index already enforces uniqueness
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private static async ValueTask<HashSet<string>> LoadAppliedMigrationsAsync(
|
||||
IMongoCollection<VexMigrationRecord> collection,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var records = await collection.Find(FilterDefinition<VexMigrationRecord>.Empty)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(static record => record.Id)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
internal sealed class VexObservationCollectionsMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251117-observations-linksets";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
await EnsureObservationsIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
await EnsureLinksetIndexesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static Task EnsureObservationsIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations);
|
||||
|
||||
var tenantObservationIndex = Builders<VexObservationRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.ObservationId);
|
||||
|
||||
var tenantVulnIndex = Builders<VexObservationRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.VulnerabilityId);
|
||||
|
||||
var tenantProductIndex = Builders<VexObservationRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.ProductKey);
|
||||
|
||||
var tenantDigestIndex = Builders<VexObservationRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending("Document.Digest");
|
||||
|
||||
var tenantProviderStatusIndex = Builders<VexObservationRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.ProviderId)
|
||||
.Ascending(x => x.Status);
|
||||
|
||||
return Task.WhenAll(
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexObservationRecord>(tenantObservationIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexObservationRecord>(tenantVulnIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexObservationRecord>(tenantProductIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexObservationRecord>(tenantDigestIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexObservationRecord>(tenantProviderStatusIndex), cancellationToken: cancellationToken));
|
||||
}
|
||||
|
||||
private static Task EnsureLinksetIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = database.GetCollection<VexLinksetRecord>(VexMongoCollectionNames.Linksets);
|
||||
|
||||
var tenantLinksetIndex = Builders<VexLinksetRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.LinksetId);
|
||||
|
||||
var tenantVulnIndex = Builders<VexLinksetRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.VulnerabilityId);
|
||||
|
||||
var tenantProductIndex = Builders<VexLinksetRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.ProductKey);
|
||||
|
||||
var tenantProviderIndex = Builders<VexLinksetRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending("ProviderIds");
|
||||
|
||||
var tenantDisagreementProviderIndex = Builders<VexLinksetRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending("Disagreements.ProviderId")
|
||||
.Ascending("Disagreements.Status");
|
||||
|
||||
return Task.WhenAll(
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantLinksetIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantVulnIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantProductIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantProviderIndex), cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(new CreateIndexModel<VexLinksetRecord>(tenantDisagreementProviderIndex), cancellationToken: cancellationToken));
|
||||
}
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
/// <summary>
|
||||
/// Adds idempotency indexes to the vex_raw collection to enforce content-addressed storage.
|
||||
/// Ensures that:
|
||||
/// 1. Each document is uniquely identified by its content digest
|
||||
/// 2. Provider+Source combinations are unique per digest
|
||||
/// 3. Supports efficient queries for evidence retrieval
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Rollback: Run db.vex_raw.dropIndex("idx_provider_sourceUri_digest_unique")
|
||||
/// and db.vex_raw.dropIndex("idx_digest_providerId") to reverse this migration.
|
||||
/// </remarks>
|
||||
internal sealed class VexRawIdempotencyIndexMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251127-vex-raw-idempotency-indexes";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
|
||||
// Index 1: Unique constraint on providerId + sourceUri + digest
|
||||
// Ensures the same document from the same provider/source is only stored once
|
||||
var providerSourceDigestIndex = new BsonDocument
|
||||
{
|
||||
{ "providerId", 1 },
|
||||
{ "sourceUri", 1 },
|
||||
{ "digest", 1 }
|
||||
};
|
||||
|
||||
var uniqueIndexModel = new CreateIndexModel<BsonDocument>(
|
||||
providerSourceDigestIndex,
|
||||
new CreateIndexOptions
|
||||
{
|
||||
Unique = true,
|
||||
Name = "idx_provider_sourceUri_digest_unique",
|
||||
Background = true
|
||||
});
|
||||
|
||||
// Index 2: Compound index for efficient evidence queries by digest + provider
|
||||
var digestProviderIndex = new BsonDocument
|
||||
{
|
||||
{ "digest", 1 },
|
||||
{ "providerId", 1 }
|
||||
};
|
||||
|
||||
var queryIndexModel = new CreateIndexModel<BsonDocument>(
|
||||
digestProviderIndex,
|
||||
new CreateIndexOptions
|
||||
{
|
||||
Name = "idx_digest_providerId",
|
||||
Background = true
|
||||
});
|
||||
|
||||
// Index 3: TTL index candidate for future cleanup (optional staleness tracking)
|
||||
var retrievedAtIndex = new BsonDocument
|
||||
{
|
||||
{ "retrievedAt", 1 }
|
||||
};
|
||||
|
||||
var retrievedAtIndexModel = new CreateIndexModel<BsonDocument>(
|
||||
retrievedAtIndex,
|
||||
new CreateIndexOptions
|
||||
{
|
||||
Name = "idx_retrievedAt",
|
||||
Background = true
|
||||
});
|
||||
|
||||
// Create all indexes
|
||||
await collection.Indexes.CreateManyAsync(
|
||||
new[] { uniqueIndexModel, queryIndexModel, retrievedAtIndexModel },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for idempotency index management.
|
||||
/// </summary>
|
||||
public static class VexRawIdempotencyIndexExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Drops the idempotency indexes (for rollback).
|
||||
/// </summary>
|
||||
public static async Task RollbackIdempotencyIndexesAsync(
|
||||
this IMongoDatabase database,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
|
||||
var indexNames = new[]
|
||||
{
|
||||
"idx_provider_sourceUri_digest_unique",
|
||||
"idx_digest_providerId",
|
||||
"idx_retrievedAt"
|
||||
};
|
||||
|
||||
foreach (var indexName in indexNames)
|
||||
{
|
||||
try
|
||||
{
|
||||
await collection.Indexes.DropOneAsync(indexName, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (MongoCommandException ex) when (ex.CodeName == "IndexNotFound")
|
||||
{
|
||||
// Index doesn't exist, skip
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that idempotency indexes exist.
|
||||
/// </summary>
|
||||
public static async Task<bool> VerifyIdempotencyIndexesExistAsync(
|
||||
this IMongoDatabase database,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var cursor = await collection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false);
|
||||
var indexes = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var indexNames = indexes.Select(i => i.GetValue("name", "").AsString).ToHashSet();
|
||||
|
||||
return indexNames.Contains("idx_provider_sourceUri_digest_unique") &&
|
||||
indexNames.Contains("idx_digest_providerId");
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
/// <summary>
|
||||
/// Adds a $jsonSchema validator to the raw VEX collection to enforce aggregation-only
|
||||
/// shape (immutable content hash + provenance fields).
|
||||
/// ValidationAction=warn keeps rollout safe while surfacing violations.
|
||||
/// </summary>
|
||||
internal sealed class VexRawSchemaMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251125-vex-raw-json-schema";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var exists = await CollectionExistsAsync(database, VexMongoCollectionNames.Raw, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var validator = BuildValidator();
|
||||
|
||||
if (!exists)
|
||||
{
|
||||
// In MongoDB.Driver 3.x, CreateCollectionOptions doesn't support Validator directly.
|
||||
// Use the create command instead.
|
||||
var createCommand = new BsonDocument
|
||||
{
|
||||
{ "create", VexMongoCollectionNames.Raw },
|
||||
{ "validator", validator },
|
||||
{ "validationAction", "warn" },
|
||||
{ "validationLevel", "moderate" }
|
||||
};
|
||||
await database.RunCommandAsync<BsonDocument>(createCommand, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var command = new BsonDocument
|
||||
{
|
||||
{ "collMod", VexMongoCollectionNames.Raw },
|
||||
{ "validator", validator },
|
||||
{ "validationAction", "warn" },
|
||||
{ "validationLevel", "moderate" },
|
||||
};
|
||||
|
||||
await database.RunCommandAsync<BsonDocument>(command, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static async Task<bool> CollectionExistsAsync(
|
||||
IMongoDatabase database,
|
||||
string name,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var cursor = await database.ListCollectionNamesAsync(
|
||||
new ListCollectionNamesOptions { Filter = new BsonDocument("name", name) },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return await cursor.AnyAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static BsonDocument BuildValidator()
|
||||
{
|
||||
var properties = new BsonDocument
|
||||
{
|
||||
{ "_id", new BsonDocument { { "bsonType", "string" }, { "description", "digest" } } },
|
||||
{ "providerId", new BsonDocument { { "bsonType", "string" }, { "minLength", 1 } } },
|
||||
{ "format", new BsonDocument
|
||||
{
|
||||
{ "bsonType", "string" },
|
||||
{ "enum", new BsonArray { "csaf", "cyclonedx", "openvex" } }
|
||||
}
|
||||
},
|
||||
{ "sourceUri", new BsonDocument { { "bsonType", "string" }, { "minLength", 1 } } },
|
||||
{ "retrievedAt", new BsonDocument { { "bsonType", "date" } } },
|
||||
{ "digest", new BsonDocument { { "bsonType", "string" }, { "minLength", 32 } } },
|
||||
{ "content", new BsonDocument
|
||||
{
|
||||
{ "bsonType", new BsonArray { "binData", "string" } }
|
||||
}
|
||||
},
|
||||
{ "gridFsObjectId", new BsonDocument
|
||||
{
|
||||
{ "bsonType", new BsonArray { "objectId", "null", "string" } }
|
||||
}
|
||||
},
|
||||
{ "metadata", new BsonDocument
|
||||
{
|
||||
{ "bsonType", "object" },
|
||||
{ "additionalProperties", true },
|
||||
{ "patternProperties", new BsonDocument
|
||||
{
|
||||
{ ".*", new BsonDocument { { "bsonType", "string" } } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return new BsonDocument
|
||||
{
|
||||
{
|
||||
"$jsonSchema",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "bsonType", "object" },
|
||||
{ "required", new BsonArray { "_id", "providerId", "format", "sourceUri", "retrievedAt", "digest" } },
|
||||
{ "properties", properties },
|
||||
{ "additionalProperties", true }
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
/// <summary>
|
||||
/// Migration that creates indexes for the vex.timeline_events collection.
|
||||
/// </summary>
|
||||
internal sealed class VexTimelineEventIndexMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251127-timeline-events";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<VexTimelineEventRecord>(VexMongoCollectionNames.TimelineEvents);
|
||||
|
||||
// Unique index on tenant + event ID
|
||||
var tenantEventIdIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.Id);
|
||||
|
||||
// Index for querying by time range (descending for recent-first queries)
|
||||
var tenantTimeIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Descending(x => x.CreatedAt);
|
||||
|
||||
// Index for querying by trace ID
|
||||
var tenantTraceIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.TraceId)
|
||||
.Ascending(x => x.CreatedAt);
|
||||
|
||||
// Index for querying by provider
|
||||
var tenantProviderIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.ProviderId)
|
||||
.Descending(x => x.CreatedAt);
|
||||
|
||||
// Index for querying by event type
|
||||
var tenantEventTypeIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.EventType)
|
||||
.Descending(x => x.CreatedAt);
|
||||
|
||||
// TTL index for automatic cleanup (30 days by default)
|
||||
// Uncomment if timeline events should expire:
|
||||
// var ttlIndex = Builders<VexTimelineEventRecord>.IndexKeys.Ascending(x => x.CreatedAt);
|
||||
// var ttlOptions = new CreateIndexOptions { ExpireAfter = TimeSpan.FromDays(30) };
|
||||
|
||||
await Task.WhenAll(
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantEventIdIndex, new CreateIndexOptions { Unique = true }),
|
||||
cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantTimeIndex),
|
||||
cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantTraceIndex),
|
||||
cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantProviderIndex),
|
||||
cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantEventTypeIndex),
|
||||
cancellationToken: cancellationToken)
|
||||
).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexAttestationLinkStore : IVexAttestationLinkStore
|
||||
{
|
||||
private readonly IMongoCollection<VexAttestationLinkRecord> _collection;
|
||||
|
||||
public MongoVexAttestationLinkStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<VexAttestationLinkRecord>(VexMongoCollectionNames.Attestations);
|
||||
}
|
||||
|
||||
public async ValueTask UpsertAsync(VexAttestationPayload payload, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(payload);
|
||||
|
||||
var record = VexAttestationLinkRecord.FromDomain(payload);
|
||||
var filter = Builders<VexAttestationLinkRecord>.Filter.Eq(x => x.AttestationId, record.AttestationId);
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
|
||||
await _collection.ReplaceOneAsync(filter, record, options, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask<VexAttestationPayload?> FindAsync(string attestationId, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(attestationId))
|
||||
{
|
||||
throw new ArgumentException("Attestation id must be provided.", nameof(attestationId));
|
||||
}
|
||||
|
||||
var filter = Builders<VexAttestationLinkRecord>.Filter.Eq(x => x.AttestationId, attestationId.Trim());
|
||||
var record = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
return record?.ToDomain();
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexCacheIndex : IVexCacheIndex
|
||||
{
|
||||
private readonly IMongoCollection<VexCacheEntryRecord> _collection;
|
||||
|
||||
public MongoVexCacheIndex(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache);
|
||||
}
|
||||
|
||||
public async ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(signature);
|
||||
var filter = Builders<VexCacheEntryRecord>.Filter.Eq(x => x.Id, VexCacheEntryRecord.CreateId(signature, format));
|
||||
var record = session is null
|
||||
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
return record?.ToDomain();
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
var record = VexCacheEntryRecord.FromDomain(entry);
|
||||
var filter = Builders<VexCacheEntryRecord>.Filter.Eq(x => x.Id, record.Id);
|
||||
if (session is null)
|
||||
{
|
||||
await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(signature);
|
||||
var filter = Builders<VexCacheEntryRecord>.Filter.Eq(x => x.Id, VexCacheEntryRecord.CreateId(signature, format));
|
||||
if (session is null)
|
||||
{
|
||||
await _collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.DeleteOneAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
internal sealed class MongoVexCacheMaintenance : IVexCacheMaintenance
|
||||
{
|
||||
private readonly IMongoCollection<VexCacheEntryRecord> _cache;
|
||||
private readonly IMongoCollection<VexExportManifestRecord> _exports;
|
||||
private readonly ILogger<MongoVexCacheMaintenance> _logger;
|
||||
|
||||
public MongoVexCacheMaintenance(
|
||||
IMongoDatabase database,
|
||||
ILogger<MongoVexCacheMaintenance> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
VexMongoMappingRegistry.Register();
|
||||
|
||||
_cache = database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache);
|
||||
_exports = database.GetCollection<VexExportManifestRecord>(VexMongoCollectionNames.Exports);
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
var cutoff = asOf.UtcDateTime;
|
||||
var filter = Builders<VexCacheEntryRecord>.Filter.Lt(x => x.ExpiresAt, cutoff);
|
||||
DeleteResult result;
|
||||
if (session is null)
|
||||
{
|
||||
result = await _cache.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = await _cache.DeleteManyAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var removed = (int)result.DeletedCount;
|
||||
if (removed > 0)
|
||||
{
|
||||
_logger.LogInformation("Pruned {Count} expired VEX export cache entries (cutoff {Cutoff})", removed, cutoff);
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
public async ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
var filter = Builders<VexCacheEntryRecord>.Filter.Ne(x => x.ManifestId, null);
|
||||
var cursor = session is null
|
||||
? await _cache.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _cache.Find(session, filter).ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (cursor.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var danglingIds = new List<string>(cursor.Count);
|
||||
foreach (var entry in cursor)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(entry.ManifestId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var manifestFilter = Builders<VexExportManifestRecord>.Filter.Eq(x => x.Id, entry.ManifestId);
|
||||
var manifestQuery = session is null
|
||||
? _exports.Find(manifestFilter)
|
||||
: _exports.Find(session, manifestFilter);
|
||||
var manifestExists = await manifestQuery
|
||||
.Limit(1)
|
||||
.AnyAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!manifestExists)
|
||||
{
|
||||
danglingIds.Add(entry.Id);
|
||||
}
|
||||
}
|
||||
|
||||
if (danglingIds.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var danglingFilter = Builders<VexCacheEntryRecord>.Filter.In(x => x.Id, danglingIds);
|
||||
DeleteResult result;
|
||||
if (session is null)
|
||||
{
|
||||
result = await _cache.DeleteManyAsync(danglingFilter, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = await _cache.DeleteManyAsync(session, danglingFilter, options: null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var removed = (int)result.DeletedCount;
|
||||
_logger.LogWarning("Removed {Count} cache entries referencing missing export manifests.", removed);
|
||||
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexClaimStore : IVexClaimStore
|
||||
{
|
||||
private readonly IMongoCollection<VexStatementRecord> _collection;
|
||||
|
||||
public MongoVexClaimStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<VexStatementRecord>(VexMongoCollectionNames.Statements);
|
||||
}
|
||||
|
||||
public async ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(claims);
|
||||
var records = claims
|
||||
.Select(claim => VexStatementRecord.FromDomain(claim, observedAt))
|
||||
.ToList();
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (session is null)
|
||||
{
|
||||
await _collection.InsertManyAsync(records, new InsertManyOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.InsertManyAsync(session, records, new InsertManyOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(productKey);
|
||||
|
||||
var filter = Builders<VexStatementRecord>.Filter.Eq(x => x.VulnerabilityId, vulnerabilityId.Trim()) &
|
||||
Builders<VexStatementRecord>.Filter.Eq(x => x.Product.Key, productKey.Trim());
|
||||
|
||||
if (since is { } sinceValue)
|
||||
{
|
||||
filter &= Builders<VexStatementRecord>.Filter.Gte(x => x.InsertedAt, sinceValue.UtcDateTime);
|
||||
}
|
||||
|
||||
var find = session is null
|
||||
? _collection.Find(filter)
|
||||
: _collection.Find(session, filter);
|
||||
|
||||
var records = await find
|
||||
.SortByDescending(x => x.InsertedAt)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.ConvertAll(static record => record.ToDomain());
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId);
|
||||
|
||||
var filter = Builders<VexStatementRecord>.Filter.Eq(x => x.VulnerabilityId, vulnerabilityId.Trim());
|
||||
|
||||
var find = session is null
|
||||
? _collection.Find(filter)
|
||||
: _collection.Find(session, filter);
|
||||
|
||||
var records = await find
|
||||
.SortByDescending(x => x.InsertedAt)
|
||||
.Limit(limit)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.ConvertAll(static record => record.ToDomain());
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexConnectorStateRepository : IVexConnectorStateRepository
|
||||
{
|
||||
private readonly IMongoCollection<VexConnectorStateDocument> _collection;
|
||||
|
||||
public MongoVexConnectorStateRepository(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<VexConnectorStateDocument>(VexMongoCollectionNames.ConnectorState);
|
||||
}
|
||||
|
||||
public async ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(connectorId);
|
||||
|
||||
var filter = Builders<VexConnectorStateDocument>.Filter.Eq(x => x.ConnectorId, connectorId.Trim());
|
||||
var document = session is null
|
||||
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
return document?.ToRecord();
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(state);
|
||||
|
||||
var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests());
|
||||
var filter = Builders<VexConnectorStateDocument>.Filter.Eq(x => x.ConnectorId, document.ConnectorId);
|
||||
if (session is null)
|
||||
{
|
||||
await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.ReplaceOneAsync(session, filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
var find = session is null
|
||||
? _collection.Find(FilterDefinition<VexConnectorStateDocument>.Empty)
|
||||
: _collection.Find(session, FilterDefinition<VexConnectorStateDocument>.Empty);
|
||||
|
||||
var documents = await find
|
||||
.SortBy(x => x.ConnectorId)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents.ConvertAll(static document => document.ToRecord());
|
||||
}
|
||||
}
|
||||
|
||||
internal static class VexConnectorStateExtensions
|
||||
{
|
||||
private const int MaxDigestHistory = 200;
|
||||
|
||||
public static VexConnectorState WithNormalizedDigests(this VexConnectorState state)
|
||||
{
|
||||
var digests = state.DocumentDigests;
|
||||
if (digests.Length <= MaxDigestHistory)
|
||||
{
|
||||
return state;
|
||||
}
|
||||
|
||||
var trimmed = digests.Skip(digests.Length - MaxDigestHistory).ToImmutableArray();
|
||||
return state with { DocumentDigests = trimmed };
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexConsensusHoldStore : IVexConsensusHoldStore
|
||||
{
|
||||
private readonly IMongoCollection<VexConsensusHoldRecord> _collection;
|
||||
|
||||
public MongoVexConsensusHoldStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<VexConsensusHoldRecord>(VexMongoCollectionNames.ConsensusHolds);
|
||||
}
|
||||
|
||||
public async ValueTask<VexConsensusHold?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(productKey);
|
||||
var id = VexConsensusRecord.CreateId(vulnerabilityId, productKey);
|
||||
var filter = Builders<VexConsensusHoldRecord>.Filter.Eq(x => x.Id, id);
|
||||
var record = session is null
|
||||
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
return record?.ToDomain();
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(VexConsensusHold hold, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(hold);
|
||||
var record = VexConsensusHoldRecord.FromDomain(hold);
|
||||
var filter = Builders<VexConsensusHoldRecord>.Filter.Eq(x => x.Id, record.Id);
|
||||
if (session is null)
|
||||
{
|
||||
await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask RemoveAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(productKey);
|
||||
var id = VexConsensusRecord.CreateId(vulnerabilityId, productKey);
|
||||
var filter = Builders<VexConsensusHoldRecord>.Filter.Eq(x => x.Id, id);
|
||||
if (session is null)
|
||||
{
|
||||
await _collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.DeleteOneAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<VexConsensusHold> FindEligibleAsync(DateTimeOffset asOf, int batchSize, [EnumeratorCancellation] CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
var cutoff = asOf.UtcDateTime;
|
||||
var filter = Builders<VexConsensusHoldRecord>.Filter.Lte(x => x.EligibleAt, cutoff);
|
||||
var find = session is null
|
||||
? _collection.Find(filter)
|
||||
: _collection.Find(session, filter);
|
||||
|
||||
find = find.SortBy(x => x.EligibleAt);
|
||||
|
||||
if (batchSize > 0)
|
||||
{
|
||||
find = find.Limit(batchSize);
|
||||
}
|
||||
|
||||
using var cursor = await find.ToCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var record in cursor.Current)
|
||||
{
|
||||
yield return record.ToDomain();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexConsensusStore : IVexConsensusStore
|
||||
{
|
||||
private readonly IMongoCollection<VexConsensusRecord> _collection;
|
||||
|
||||
public MongoVexConsensusStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<VexConsensusRecord>(VexMongoCollectionNames.Consensus);
|
||||
}
|
||||
|
||||
public async ValueTask<VexConsensus?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(productKey);
|
||||
var id = VexConsensusRecord.CreateId(vulnerabilityId, productKey);
|
||||
var filter = Builders<VexConsensusRecord>.Filter.Eq(x => x.Id, id);
|
||||
var record = session is null
|
||||
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
return record?.ToDomain();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyCollection<VexConsensus>> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId);
|
||||
var filter = Builders<VexConsensusRecord>.Filter.Eq(x => x.VulnerabilityId, vulnerabilityId.Trim());
|
||||
var find = session is null
|
||||
? _collection.Find(filter)
|
||||
: _collection.Find(session, filter);
|
||||
var records = await find.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
return records.ConvertAll(static record => record.ToDomain());
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(consensus);
|
||||
var record = VexConsensusRecord.FromDomain(consensus);
|
||||
var filter = Builders<VexConsensusRecord>.Filter.Eq(x => x.Id, record.Id);
|
||||
if (session is null)
|
||||
{
|
||||
await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<VexConsensus> FindCalculatedBeforeAsync(DateTimeOffset cutoff, int batchSize, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
var filter = Builders<VexConsensusRecord>.Filter.Lt(x => x.CalculatedAt, cutoff.UtcDateTime);
|
||||
var find = session is null
|
||||
? _collection.Find(filter)
|
||||
: _collection.Find(session, filter);
|
||||
|
||||
find = find.SortBy(x => x.CalculatedAt);
|
||||
|
||||
if (batchSize > 0)
|
||||
{
|
||||
find = find.Limit(batchSize);
|
||||
}
|
||||
|
||||
using var cursor = await find.ToCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var record in cursor.Current)
|
||||
{
|
||||
yield return record.ToDomain();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
using System;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.Core.Clusters;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexExportStore : IVexExportStore
|
||||
{
|
||||
private readonly IMongoClient _client;
|
||||
private readonly IMongoCollection<VexExportManifestRecord> _exports;
|
||||
private readonly IMongoCollection<VexCacheEntryRecord> _cache;
|
||||
private readonly VexMongoStorageOptions _options;
|
||||
private readonly IVexMongoSessionProvider _sessionProvider;
|
||||
|
||||
public MongoVexExportStore(
|
||||
IMongoClient client,
|
||||
IMongoDatabase database,
|
||||
IOptions<VexMongoStorageOptions> options,
|
||||
IVexMongoSessionProvider sessionProvider)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
|
||||
|
||||
_options = options.Value;
|
||||
Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true);
|
||||
|
||||
VexMongoMappingRegistry.Register();
|
||||
_exports = database.GetCollection<VexExportManifestRecord>(VexMongoCollectionNames.Exports);
|
||||
_cache = database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache);
|
||||
}
|
||||
|
||||
public async ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(signature);
|
||||
var cacheId = VexCacheEntryRecord.CreateId(signature, format);
|
||||
var cacheFilter = Builders<VexCacheEntryRecord>.Filter.Eq(x => x.Id, cacheId);
|
||||
var cacheRecord = session is null
|
||||
? await _cache.Find(cacheFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _cache.Find(session, cacheFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (cacheRecord is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (cacheRecord.ExpiresAt is DateTime expiresAt && expiresAt <= DateTime.UtcNow)
|
||||
{
|
||||
await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false);
|
||||
return null;
|
||||
}
|
||||
|
||||
var manifestId = VexExportManifestRecord.CreateId(signature, format);
|
||||
var manifestFilter = Builders<VexExportManifestRecord>.Filter.Eq(x => x.Id, manifestId);
|
||||
var manifest = session is null
|
||||
? await _exports.Find(manifestFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _exports.Find(session, manifestFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (manifest is null)
|
||||
{
|
||||
if (session is null)
|
||||
{
|
||||
await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _cache.DeleteOneAsync(session, cacheFilter, options: null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(cacheRecord.ManifestId) &&
|
||||
!string.Equals(cacheRecord.ManifestId, manifest.Id, StringComparison.Ordinal))
|
||||
{
|
||||
if (session is null)
|
||||
{
|
||||
await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _cache.DeleteOneAsync(session, cacheFilter, options: null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return manifest.ToDomain();
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
|
||||
var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone
|
||||
&& !sessionHandle.IsInTransaction;
|
||||
|
||||
var startedTransaction = false;
|
||||
if (supportsTransactions)
|
||||
{
|
||||
try
|
||||
{
|
||||
sessionHandle.StartTransaction();
|
||||
startedTransaction = true;
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
supportsTransactions = false;
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var manifestRecord = VexExportManifestRecord.FromDomain(manifest);
|
||||
var manifestFilter = Builders<VexExportManifestRecord>.Filter.Eq(x => x.Id, manifestRecord.Id);
|
||||
|
||||
await _exports
|
||||
.ReplaceOneAsync(
|
||||
sessionHandle,
|
||||
manifestFilter,
|
||||
manifestRecord,
|
||||
new ReplaceOptions { IsUpsert = true },
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var cacheEntry = CreateCacheEntry(manifest);
|
||||
var cacheRecord = VexCacheEntryRecord.FromDomain(cacheEntry);
|
||||
var cacheFilter = Builders<VexCacheEntryRecord>.Filter.Eq(x => x.Id, cacheRecord.Id);
|
||||
|
||||
await _cache
|
||||
.ReplaceOneAsync(
|
||||
sessionHandle,
|
||||
cacheFilter,
|
||||
cacheRecord,
|
||||
new ReplaceOptions { IsUpsert = true },
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (startedTransaction)
|
||||
{
|
||||
await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (startedTransaction && sessionHandle.IsInTransaction)
|
||||
{
|
||||
await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private VexCacheEntry CreateCacheEntry(VexExportManifest manifest)
|
||||
{
|
||||
var expiresAt = manifest.CreatedAt + _options.ExportCacheTtl;
|
||||
return new VexCacheEntry(
|
||||
manifest.QuerySignature,
|
||||
manifest.Format,
|
||||
manifest.Artifact,
|
||||
manifest.CreatedAt,
|
||||
manifest.SizeBytes,
|
||||
manifestId: manifest.ExportId,
|
||||
gridFsObjectId: null,
|
||||
expiresAt: expiresAt);
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB implementation of <see cref="IVexLinksetEventPublisher"/>.
|
||||
/// Events are persisted to the vex.linkset_events collection for replay and audit.
|
||||
/// </summary>
|
||||
internal sealed class MongoVexLinksetEventPublisher : IVexLinksetEventPublisher
|
||||
{
|
||||
private readonly IMongoCollection<VexLinksetEventRecord> _collection;
|
||||
|
||||
public MongoVexLinksetEventPublisher(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexLinksetEventRecord>(VexMongoCollectionNames.LinksetEvents);
|
||||
}
|
||||
|
||||
public async Task PublishAsync(VexLinksetUpdatedEvent @event, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
|
||||
var record = ToRecord(@event);
|
||||
await _collection.InsertOneAsync(record, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task PublishManyAsync(IEnumerable<VexLinksetUpdatedEvent> events, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(events);
|
||||
|
||||
var records = events
|
||||
.Where(e => e is not null)
|
||||
.Select(ToRecord)
|
||||
.ToList();
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = new InsertManyOptions { IsOrdered = false };
|
||||
await _collection.InsertManyAsync(records, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static VexLinksetEventRecord ToRecord(VexLinksetUpdatedEvent @event)
|
||||
{
|
||||
var eventId = $"{@event.LinksetId}:{@event.CreatedAtUtc.UtcTicks}";
|
||||
|
||||
return new VexLinksetEventRecord
|
||||
{
|
||||
Id = eventId,
|
||||
EventType = @event.EventType,
|
||||
Tenant = @event.Tenant.ToLowerInvariant(),
|
||||
LinksetId = @event.LinksetId,
|
||||
VulnerabilityId = @event.VulnerabilityId,
|
||||
ProductKey = @event.ProductKey,
|
||||
Scope = new VexLinksetScopeRecord
|
||||
{
|
||||
ProductKey = @event.Scope.ProductKey,
|
||||
Type = @event.Scope.Type,
|
||||
Version = @event.Scope.Version,
|
||||
Purl = @event.Scope.Purl,
|
||||
Cpe = @event.Scope.Cpe,
|
||||
Identifiers = @event.Scope.Identifiers.ToList(),
|
||||
},
|
||||
Observations = @event.Observations
|
||||
.Select(o => new VexLinksetEventObservationRecord
|
||||
{
|
||||
ObservationId = o.ObservationId,
|
||||
ProviderId = o.ProviderId,
|
||||
Status = o.Status,
|
||||
Confidence = o.Confidence
|
||||
})
|
||||
.ToList(),
|
||||
Disagreements = @event.Disagreements
|
||||
.Select(d => new VexLinksetDisagreementRecord
|
||||
{
|
||||
ProviderId = d.ProviderId,
|
||||
Status = d.Status,
|
||||
Justification = d.Justification,
|
||||
Confidence = d.Confidence
|
||||
})
|
||||
.ToList(),
|
||||
CreatedAtUtc = @event.CreatedAtUtc.UtcDateTime,
|
||||
PublishedAtUtc = DateTime.UtcNow,
|
||||
ConflictCount = @event.Disagreements.Length,
|
||||
ObservationCount = @event.Observations.Length
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,376 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
internal sealed class MongoVexLinksetStore : IVexLinksetStore
|
||||
{
|
||||
private readonly IMongoCollection<VexLinksetRecord> _collection;
|
||||
|
||||
public MongoVexLinksetStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexLinksetRecord>(VexMongoCollectionNames.Linksets);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> InsertAsync(
|
||||
VexLinkset linkset,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(linkset);
|
||||
|
||||
var record = ToRecord(linkset);
|
||||
|
||||
try
|
||||
{
|
||||
await _collection.InsertOneAsync(record, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<bool> UpsertAsync(
|
||||
VexLinkset linkset,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(linkset);
|
||||
|
||||
var record = ToRecord(linkset);
|
||||
var normalizedTenant = NormalizeTenant(linkset.Tenant);
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.LinksetId, linkset.LinksetId));
|
||||
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
var result = await _collection
|
||||
.ReplaceOneAsync(filter, record, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return result.UpsertedId is not null;
|
||||
}
|
||||
|
||||
public async ValueTask<VexLinkset?> GetByIdAsync(
|
||||
string tenant,
|
||||
string linksetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = linksetId?.Trim() ?? throw new ArgumentNullException(nameof(linksetId));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.LinksetId, normalizedId));
|
||||
|
||||
var record = await _collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return record is null ? null : ToModel(record);
|
||||
}
|
||||
|
||||
public async ValueTask<VexLinkset> GetOrCreateAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedVuln = vulnerabilityId?.Trim() ?? throw new ArgumentNullException(nameof(vulnerabilityId));
|
||||
var normalizedProduct = productKey?.Trim() ?? throw new ArgumentNullException(nameof(productKey));
|
||||
|
||||
var linksetId = VexLinkset.CreateLinksetId(normalizedTenant, normalizedVuln, normalizedProduct);
|
||||
|
||||
var existing = await GetByIdAsync(normalizedTenant, linksetId, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null)
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
var newLinkset = new VexLinkset(
|
||||
linksetId,
|
||||
normalizedTenant,
|
||||
normalizedVuln,
|
||||
normalizedProduct,
|
||||
scope: VexProductScope.Unknown(normalizedProduct),
|
||||
observations: Array.Empty<VexLinksetObservationRefModel>(),
|
||||
disagreements: null,
|
||||
createdAt: DateTimeOffset.UtcNow,
|
||||
updatedAt: DateTimeOffset.UtcNow);
|
||||
|
||||
try
|
||||
{
|
||||
await InsertAsync(newLinkset, cancellationToken).ConfigureAwait(false);
|
||||
return newLinkset;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
// Race condition - another process created it. Fetch and return.
|
||||
var created = await GetByIdAsync(normalizedTenant, linksetId, cancellationToken).ConfigureAwait(false);
|
||||
return created ?? newLinkset;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexLinkset>> FindByVulnerabilityAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedVuln = vulnerabilityId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(vulnerabilityId));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.VulnerabilityId, normalizedVuln));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexLinksetRecord>.Sort.Descending(r => r.UpdatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexLinkset>> FindByProductKeyAsync(
|
||||
string tenant,
|
||||
string productKey,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedProduct = productKey?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(productKey));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.ProductKey, normalizedProduct));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexLinksetRecord>.Sort.Descending(r => r.UpdatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexLinkset>> FindWithConflictsAsync(
|
||||
string tenant,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.SizeGt(r => r.Disagreements, 0));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexLinksetRecord>.Sort.Descending(r => r.UpdatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexLinkset>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedProvider = providerId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(providerId));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.AnyEq(r => r.ProviderIds, normalizedProvider));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexLinksetRecord>.Sort.Descending(r => r.UpdatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<bool> DeleteAsync(
|
||||
string tenant,
|
||||
string linksetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = linksetId?.Trim() ?? throw new ArgumentNullException(nameof(linksetId));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.LinksetId, normalizedId));
|
||||
|
||||
var result = await _collection
|
||||
.DeleteOneAsync(filter, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return result.DeletedCount > 0;
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant);
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountWithConflictsAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.SizeGt(r => r.Disagreements, 0));
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string NormalizeTenant(string tenant)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static VexLinksetRecord ToRecord(VexLinkset linkset)
|
||||
{
|
||||
return new VexLinksetRecord
|
||||
{
|
||||
Id = linkset.LinksetId,
|
||||
Tenant = linkset.Tenant.ToLowerInvariant(),
|
||||
LinksetId = linkset.LinksetId,
|
||||
VulnerabilityId = linkset.VulnerabilityId.ToLowerInvariant(),
|
||||
ProductKey = linkset.ProductKey.ToLowerInvariant(),
|
||||
Scope = ToScopeRecord(linkset.Scope),
|
||||
ProviderIds = linkset.ProviderIds.ToList(),
|
||||
Statuses = linkset.Statuses.ToList(),
|
||||
CreatedAt = linkset.CreatedAt.UtcDateTime,
|
||||
UpdatedAt = linkset.UpdatedAt.UtcDateTime,
|
||||
Observations = linkset.Observations.Select(ToObservationRecord).ToList(),
|
||||
Disagreements = linkset.Disagreements.Select(ToDisagreementRecord).ToList()
|
||||
};
|
||||
}
|
||||
|
||||
private static VexObservationLinksetObservationRecord ToObservationRecord(VexLinksetObservationRefModel obs)
|
||||
{
|
||||
return new VexObservationLinksetObservationRecord
|
||||
{
|
||||
ObservationId = obs.ObservationId,
|
||||
ProviderId = obs.ProviderId,
|
||||
Status = obs.Status,
|
||||
Confidence = obs.Confidence
|
||||
};
|
||||
}
|
||||
|
||||
private static VexLinksetDisagreementRecord ToDisagreementRecord(VexObservationDisagreement disagreement)
|
||||
{
|
||||
return new VexLinksetDisagreementRecord
|
||||
{
|
||||
ProviderId = disagreement.ProviderId,
|
||||
Status = disagreement.Status,
|
||||
Justification = disagreement.Justification,
|
||||
Confidence = disagreement.Confidence
|
||||
};
|
||||
}
|
||||
|
||||
private static VexLinkset ToModel(VexLinksetRecord record)
|
||||
{
|
||||
var observations = record.Observations?
|
||||
.Where(o => o is not null)
|
||||
.Select(o => new VexLinksetObservationRefModel(
|
||||
o.ObservationId,
|
||||
o.ProviderId,
|
||||
o.Status,
|
||||
o.Confidence))
|
||||
.ToImmutableArray() ?? ImmutableArray<VexLinksetObservationRefModel>.Empty;
|
||||
|
||||
var disagreements = record.Disagreements?
|
||||
.Where(d => d is not null)
|
||||
.Select(d => new VexObservationDisagreement(
|
||||
d.ProviderId,
|
||||
d.Status,
|
||||
d.Justification,
|
||||
d.Confidence))
|
||||
.ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
|
||||
var scope = record.Scope is not null
|
||||
? ToScope(record.Scope)
|
||||
: VexProductScope.Unknown(record.ProductKey);
|
||||
|
||||
return new VexLinkset(
|
||||
linksetId: record.LinksetId,
|
||||
tenant: record.Tenant,
|
||||
vulnerabilityId: record.VulnerabilityId,
|
||||
productKey: record.ProductKey,
|
||||
scope: scope,
|
||||
observations: observations,
|
||||
disagreements: disagreements,
|
||||
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
|
||||
updatedAt: new DateTimeOffset(record.UpdatedAt, TimeSpan.Zero));
|
||||
}
|
||||
|
||||
private static VexLinksetScopeRecord ToScopeRecord(VexProductScope scope)
|
||||
{
|
||||
return new VexLinksetScopeRecord
|
||||
{
|
||||
ProductKey = scope.ProductKey,
|
||||
Type = scope.Type,
|
||||
Version = scope.Version,
|
||||
Purl = scope.Purl,
|
||||
Cpe = scope.Cpe,
|
||||
Identifiers = scope.Identifiers.ToList()
|
||||
};
|
||||
}
|
||||
|
||||
private static VexProductScope ToScope(VexLinksetScopeRecord record)
|
||||
{
|
||||
var identifiers = record.Identifiers?
|
||||
.Where(id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(id => id.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
|
||||
return new VexProductScope(
|
||||
ProductKey: record.ProductKey ?? string.Empty,
|
||||
Type: record.Type ?? "unknown",
|
||||
Version: record.Version,
|
||||
Purl: record.Purl,
|
||||
Cpe: record.Cpe,
|
||||
Identifiers: identifiers);
|
||||
}
|
||||
}
|
||||
@@ -1,222 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
internal sealed class MongoVexObservationLookup : IVexObservationLookup
|
||||
{
|
||||
private readonly IMongoCollection<VexObservationRecord> _collection;
|
||||
|
||||
public MongoVexObservationLookup(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexObservation>> ListByTenantAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.Eq(record => record.Tenant, normalizedTenant);
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt).Descending(r => r.ObservationId))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(Map).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexObservation>> FindByFiltersAsync(
|
||||
string tenant,
|
||||
IReadOnlyCollection<string> observationIds,
|
||||
IReadOnlyCollection<string> vulnerabilityIds,
|
||||
IReadOnlyCollection<string> productKeys,
|
||||
IReadOnlyCollection<string> purls,
|
||||
IReadOnlyCollection<string> cpes,
|
||||
IReadOnlyCollection<string> providerIds,
|
||||
IReadOnlyCollection<VexClaimStatus> statuses,
|
||||
VexObservationCursor? cursor,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filters = new List<FilterDefinition<VexObservationRecord>>(capacity: 8)
|
||||
{
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant)
|
||||
};
|
||||
|
||||
AddInFilter(filters, r => r.ObservationId, observationIds);
|
||||
AddInFilter(filters, r => r.VulnerabilityId, vulnerabilityIds.Select(v => v.ToLowerInvariant()));
|
||||
AddInFilter(filters, r => r.ProductKey, productKeys.Select(p => p.ToLowerInvariant()));
|
||||
AddInFilter(filters, r => r.ProviderId, providerIds.Select(p => p.ToLowerInvariant()));
|
||||
|
||||
if (statuses.Count > 0)
|
||||
{
|
||||
var statusStrings = statuses.Select(status => status.ToString().ToLowerInvariant()).ToArray();
|
||||
filters.Add(Builders<VexObservationRecord>.Filter.In(r => r.Status, statusStrings));
|
||||
}
|
||||
|
||||
if (cursor is not null)
|
||||
{
|
||||
var cursorFilter = Builders<VexObservationRecord>.Filter.Or(
|
||||
Builders<VexObservationRecord>.Filter.Lt(r => r.CreatedAt, cursor.CreatedAt.UtcDateTime),
|
||||
Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.CreatedAt, cursor.CreatedAt.UtcDateTime),
|
||||
Builders<VexObservationRecord>.Filter.Lt(r => r.ObservationId, cursor.ObservationId)));
|
||||
filters.Add(cursorFilter);
|
||||
}
|
||||
|
||||
var combinedFilter = filters.Count == 1
|
||||
? filters[0]
|
||||
: Builders<VexObservationRecord>.Filter.And(filters);
|
||||
|
||||
var records = await _collection
|
||||
.Find(combinedFilter)
|
||||
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt).Descending(r => r.ObservationId))
|
||||
.Limit(limit)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(Map).ToList();
|
||||
}
|
||||
|
||||
private static void AddInFilter(
|
||||
ICollection<FilterDefinition<VexObservationRecord>> filters,
|
||||
System.Linq.Expressions.Expression<Func<VexObservationRecord, string>> field,
|
||||
IEnumerable<string> values)
|
||||
{
|
||||
var normalized = values
|
||||
.Where(value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(value => value.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
if (normalized.Length > 0)
|
||||
{
|
||||
filters.Add(Builders<VexObservationRecord>.Filter.In(field, normalized));
|
||||
}
|
||||
}
|
||||
|
||||
private static VexObservation Map(VexObservationRecord record)
|
||||
{
|
||||
var statements = record.Statements.Select(MapStatement).ToImmutableArray();
|
||||
|
||||
var linkset = MapLinkset(record.Linkset);
|
||||
|
||||
var upstreamSignature = record.Upstream?.Signature is null
|
||||
? new VexObservationSignature(false, null, null, null)
|
||||
: new VexObservationSignature(
|
||||
record.Upstream.Signature.Present,
|
||||
record.Upstream.Signature.Subject,
|
||||
record.Upstream.Signature.Issuer,
|
||||
signature: null);
|
||||
|
||||
var upstream = record.Upstream is null
|
||||
? new VexObservationUpstream(
|
||||
upstreamId: record.ObservationId,
|
||||
documentVersion: null,
|
||||
fetchedAt: record.CreatedAt,
|
||||
receivedAt: record.CreatedAt,
|
||||
contentHash: record.Document.Digest,
|
||||
signature: upstreamSignature)
|
||||
: new VexObservationUpstream(
|
||||
record.Upstream.UpstreamId,
|
||||
record.Upstream.DocumentVersion,
|
||||
record.Upstream.FetchedAt,
|
||||
record.Upstream.ReceivedAt,
|
||||
record.Upstream.ContentHash,
|
||||
upstreamSignature);
|
||||
|
||||
var documentSignature = record.Document.Signature is null
|
||||
? null
|
||||
: new VexObservationSignature(
|
||||
record.Document.Signature.Present,
|
||||
record.Document.Signature.Subject,
|
||||
record.Document.Signature.Issuer,
|
||||
signature: null);
|
||||
|
||||
var content = record.Content is null
|
||||
? new VexObservationContent("unknown", null, new JsonObject())
|
||||
: new VexObservationContent(
|
||||
record.Content.Format ?? "unknown",
|
||||
record.Content.SpecVersion,
|
||||
JsonNode.Parse(record.Content.Raw.ToJson()) ?? new JsonObject(),
|
||||
metadata: ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
var observation = new VexObservation(
|
||||
observationId: record.ObservationId,
|
||||
tenant: record.Tenant,
|
||||
providerId: record.ProviderId,
|
||||
streamId: string.IsNullOrWhiteSpace(record.StreamId) ? record.ProviderId : record.StreamId,
|
||||
upstream: upstream,
|
||||
statements: statements,
|
||||
content: content,
|
||||
linkset: linkset,
|
||||
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
|
||||
supersedes: ImmutableArray<string>.Empty,
|
||||
attributes: ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
return observation;
|
||||
}
|
||||
|
||||
private static VexObservationStatement MapStatement(VexObservationStatementRecord record)
|
||||
{
|
||||
var justification = string.IsNullOrWhiteSpace(record.Justification)
|
||||
? (VexJustification?)null
|
||||
: Enum.Parse<VexJustification>(record.Justification, ignoreCase: true);
|
||||
|
||||
return new VexObservationStatement(
|
||||
record.VulnerabilityId,
|
||||
record.ProductKey,
|
||||
Enum.Parse<VexClaimStatus>(record.Status, ignoreCase: true),
|
||||
record.LastObserved,
|
||||
locator: record.Locator,
|
||||
justification: justification,
|
||||
introducedVersion: record.IntroducedVersion,
|
||||
fixedVersion: record.FixedVersion,
|
||||
purl: null,
|
||||
cpe: null,
|
||||
evidence: null,
|
||||
metadata: ImmutableDictionary<string, string>.Empty);
|
||||
}
|
||||
|
||||
private static VexObservationDisagreement MapDisagreement(VexLinksetDisagreementRecord record)
|
||||
=> new(record.ProviderId, record.Status, record.Justification, record.Confidence);
|
||||
|
||||
private static VexObservationLinkset MapLinkset(VexObservationLinksetRecord record)
|
||||
{
|
||||
var aliases = record?.Aliases?.Where(NotNullOrWhiteSpace).Select(a => a.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var purls = record?.Purls?.Where(NotNullOrWhiteSpace).Select(p => p.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var cpes = record?.Cpes?.Where(NotNullOrWhiteSpace).Select(c => c.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var references = record?.References?.Select(r => new VexObservationReference(r.Type, r.Url)).ToImmutableArray() ?? ImmutableArray<VexObservationReference>.Empty;
|
||||
var reconciledFrom = record?.ReconciledFrom?.Where(NotNullOrWhiteSpace).Select(r => r.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var disagreements = record?.Disagreements?.Select(MapDisagreement).ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
var observationRefs = record?.Observations?.Select(o => new VexLinksetObservationRefModel(
|
||||
o.ObservationId,
|
||||
o.ProviderId,
|
||||
o.Status,
|
||||
o.Confidence)).ToImmutableArray() ?? ImmutableArray<VexLinksetObservationRefModel>.Empty;
|
||||
|
||||
return new VexObservationLinkset(aliases, purls, cpes, references, reconciledFrom, disagreements, observationRefs);
|
||||
}
|
||||
|
||||
private static bool NotNullOrWhiteSpace(string? value) => !string.IsNullOrWhiteSpace(value);
|
||||
|
||||
private static string NormalizeTenant(string tenant)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -1,398 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
internal sealed class MongoVexObservationStore : IVexObservationStore
|
||||
{
|
||||
private readonly IMongoCollection<VexObservationRecord> _collection;
|
||||
|
||||
public MongoVexObservationStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> InsertAsync(
|
||||
VexObservation observation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(observation);
|
||||
|
||||
var record = ToRecord(observation);
|
||||
|
||||
try
|
||||
{
|
||||
await _collection.InsertOneAsync(record, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<bool> UpsertAsync(
|
||||
VexObservation observation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(observation);
|
||||
|
||||
var record = ToRecord(observation);
|
||||
var normalizedTenant = NormalizeTenant(observation.Tenant);
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ObservationId, observation.ObservationId));
|
||||
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
var result = await _collection
|
||||
.ReplaceOneAsync(filter, record, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return result.UpsertedId is not null;
|
||||
}
|
||||
|
||||
public async ValueTask<int> InsertManyAsync(
|
||||
string tenant,
|
||||
IEnumerable<VexObservation> observations,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var records = observations
|
||||
.Where(o => o is not null && string.Equals(NormalizeTenant(o.Tenant), normalizedTenant, StringComparison.Ordinal))
|
||||
.Select(ToRecord)
|
||||
.ToList();
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var options = new InsertManyOptions { IsOrdered = false };
|
||||
try
|
||||
{
|
||||
await _collection.InsertManyAsync(records, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return records.Count;
|
||||
}
|
||||
catch (MongoBulkWriteException<VexObservationRecord> ex)
|
||||
{
|
||||
// Return the count of successful inserts
|
||||
var duplicates = ex.WriteErrors?.Count(e => e.Category == ServerErrorCategory.DuplicateKey) ?? 0;
|
||||
return records.Count - duplicates;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<VexObservation?> GetByIdAsync(
|
||||
string tenant,
|
||||
string observationId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = observationId?.Trim() ?? throw new ArgumentNullException(nameof(observationId));
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ObservationId, normalizedId));
|
||||
|
||||
var record = await _collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return record is null ? null : ToModel(record);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexObservation>> FindByVulnerabilityAndProductAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedVuln = vulnerabilityId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(vulnerabilityId));
|
||||
var normalizedProduct = productKey?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(productKey));
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.VulnerabilityId, normalizedVuln),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ProductKey, normalizedProduct));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexObservation>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedProvider = providerId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(providerId));
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ProviderId, normalizedProvider));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<bool> DeleteAsync(
|
||||
string tenant,
|
||||
string observationId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = observationId?.Trim() ?? throw new ArgumentNullException(nameof(observationId));
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ObservationId, normalizedId));
|
||||
|
||||
var result = await _collection
|
||||
.DeleteOneAsync(filter, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return result.DeletedCount > 0;
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant);
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string NormalizeTenant(string tenant)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static VexObservationRecord ToRecord(VexObservation observation)
|
||||
{
|
||||
var firstStatement = observation.Statements.FirstOrDefault();
|
||||
|
||||
return new VexObservationRecord
|
||||
{
|
||||
Id = observation.ObservationId,
|
||||
Tenant = observation.Tenant,
|
||||
ObservationId = observation.ObservationId,
|
||||
VulnerabilityId = firstStatement?.VulnerabilityId?.ToLowerInvariant() ?? string.Empty,
|
||||
ProductKey = firstStatement?.ProductKey?.ToLowerInvariant() ?? string.Empty,
|
||||
ProviderId = observation.ProviderId,
|
||||
StreamId = observation.StreamId,
|
||||
Status = firstStatement?.Status.ToString().ToLowerInvariant() ?? "unknown",
|
||||
Document = new VexObservationDocumentRecord
|
||||
{
|
||||
Digest = observation.Upstream.ContentHash,
|
||||
SourceUri = null,
|
||||
Format = observation.Content.Format,
|
||||
Revision = observation.Upstream.DocumentVersion,
|
||||
Signature = new VexObservationSignatureRecord
|
||||
{
|
||||
Present = observation.Upstream.Signature.Present,
|
||||
Subject = observation.Upstream.Signature.Format,
|
||||
Issuer = observation.Upstream.Signature.KeyId,
|
||||
VerifiedAt = null
|
||||
}
|
||||
},
|
||||
Upstream = new VexObservationUpstreamRecord
|
||||
{
|
||||
UpstreamId = observation.Upstream.UpstreamId,
|
||||
DocumentVersion = observation.Upstream.DocumentVersion,
|
||||
FetchedAt = observation.Upstream.FetchedAt,
|
||||
ReceivedAt = observation.Upstream.ReceivedAt,
|
||||
ContentHash = observation.Upstream.ContentHash,
|
||||
Signature = new VexObservationSignatureRecord
|
||||
{
|
||||
Present = observation.Upstream.Signature.Present,
|
||||
Subject = observation.Upstream.Signature.Format,
|
||||
Issuer = observation.Upstream.Signature.KeyId,
|
||||
VerifiedAt = null
|
||||
}
|
||||
},
|
||||
Content = new VexObservationContentRecord
|
||||
{
|
||||
Format = observation.Content.Format,
|
||||
SpecVersion = observation.Content.SpecVersion,
|
||||
Raw = BsonDocument.Parse(observation.Content.Raw.ToJsonString())
|
||||
},
|
||||
Statements = observation.Statements.Select(ToStatementRecord).ToList(),
|
||||
Linkset = ToLinksetRecord(observation.Linkset),
|
||||
CreatedAt = observation.CreatedAt.UtcDateTime
|
||||
};
|
||||
}
|
||||
|
||||
private static VexObservationStatementRecord ToStatementRecord(VexObservationStatement statement)
|
||||
{
|
||||
return new VexObservationStatementRecord
|
||||
{
|
||||
VulnerabilityId = statement.VulnerabilityId,
|
||||
ProductKey = statement.ProductKey,
|
||||
Status = statement.Status.ToString().ToLowerInvariant(),
|
||||
LastObserved = statement.LastObserved,
|
||||
Locator = statement.Locator,
|
||||
Justification = statement.Justification?.ToString().ToLowerInvariant(),
|
||||
IntroducedVersion = statement.IntroducedVersion,
|
||||
FixedVersion = statement.FixedVersion,
|
||||
Detail = null,
|
||||
ScopeScore = null,
|
||||
Epss = null,
|
||||
Kev = null
|
||||
};
|
||||
}
|
||||
|
||||
private static VexObservationLinksetRecord ToLinksetRecord(VexObservationLinkset linkset)
|
||||
{
|
||||
return new VexObservationLinksetRecord
|
||||
{
|
||||
Aliases = linkset.Aliases.ToList(),
|
||||
Purls = linkset.Purls.ToList(),
|
||||
Cpes = linkset.Cpes.ToList(),
|
||||
References = linkset.References.Select(r => new VexObservationReferenceRecord
|
||||
{
|
||||
Type = r.Type,
|
||||
Url = r.Url
|
||||
}).ToList(),
|
||||
ReconciledFrom = linkset.ReconciledFrom.ToList(),
|
||||
Disagreements = linkset.Disagreements.Select(d => new VexLinksetDisagreementRecord
|
||||
{
|
||||
ProviderId = d.ProviderId,
|
||||
Status = d.Status,
|
||||
Justification = d.Justification,
|
||||
Confidence = d.Confidence
|
||||
}).ToList(),
|
||||
Observations = linkset.Observations.Select(o => new VexObservationLinksetObservationRecord
|
||||
{
|
||||
ObservationId = o.ObservationId,
|
||||
ProviderId = o.ProviderId,
|
||||
Status = o.Status,
|
||||
Confidence = o.Confidence
|
||||
}).ToList()
|
||||
};
|
||||
}
|
||||
|
||||
private static VexObservation ToModel(VexObservationRecord record)
|
||||
{
|
||||
var statements = record.Statements.Select(MapStatement).ToImmutableArray();
|
||||
var linkset = MapLinkset(record.Linkset);
|
||||
|
||||
var upstreamSignature = record.Upstream?.Signature is null
|
||||
? new VexObservationSignature(false, null, null, null)
|
||||
: new VexObservationSignature(
|
||||
record.Upstream.Signature.Present,
|
||||
record.Upstream.Signature.Subject,
|
||||
record.Upstream.Signature.Issuer,
|
||||
signature: null);
|
||||
|
||||
var upstream = record.Upstream is null
|
||||
? new VexObservationUpstream(
|
||||
upstreamId: record.ObservationId,
|
||||
documentVersion: null,
|
||||
fetchedAt: record.CreatedAt,
|
||||
receivedAt: record.CreatedAt,
|
||||
contentHash: record.Document.Digest,
|
||||
signature: upstreamSignature)
|
||||
: new VexObservationUpstream(
|
||||
record.Upstream.UpstreamId,
|
||||
record.Upstream.DocumentVersion,
|
||||
record.Upstream.FetchedAt,
|
||||
record.Upstream.ReceivedAt,
|
||||
record.Upstream.ContentHash,
|
||||
upstreamSignature);
|
||||
|
||||
var content = record.Content is null
|
||||
? new VexObservationContent("unknown", null, new JsonObject())
|
||||
: new VexObservationContent(
|
||||
record.Content.Format ?? "unknown",
|
||||
record.Content.SpecVersion,
|
||||
JsonNode.Parse(record.Content.Raw.ToJson()) ?? new JsonObject(),
|
||||
metadata: ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
return new VexObservation(
|
||||
observationId: record.ObservationId,
|
||||
tenant: record.Tenant,
|
||||
providerId: record.ProviderId,
|
||||
streamId: string.IsNullOrWhiteSpace(record.StreamId) ? record.ProviderId : record.StreamId,
|
||||
upstream: upstream,
|
||||
statements: statements,
|
||||
content: content,
|
||||
linkset: linkset,
|
||||
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
|
||||
supersedes: ImmutableArray<string>.Empty,
|
||||
attributes: ImmutableDictionary<string, string>.Empty);
|
||||
}
|
||||
|
||||
private static VexObservationStatement MapStatement(VexObservationStatementRecord record)
|
||||
{
|
||||
var justification = string.IsNullOrWhiteSpace(record.Justification)
|
||||
? (VexJustification?)null
|
||||
: Enum.Parse<VexJustification>(record.Justification, ignoreCase: true);
|
||||
|
||||
return new VexObservationStatement(
|
||||
record.VulnerabilityId,
|
||||
record.ProductKey,
|
||||
Enum.Parse<VexClaimStatus>(record.Status, ignoreCase: true),
|
||||
record.LastObserved,
|
||||
locator: record.Locator,
|
||||
justification: justification,
|
||||
introducedVersion: record.IntroducedVersion,
|
||||
fixedVersion: record.FixedVersion,
|
||||
purl: null,
|
||||
cpe: null,
|
||||
evidence: null,
|
||||
metadata: ImmutableDictionary<string, string>.Empty);
|
||||
}
|
||||
|
||||
private static VexObservationLinkset MapLinkset(VexObservationLinksetRecord record)
|
||||
{
|
||||
var aliases = record?.Aliases?.Where(NotNullOrWhiteSpace).Select(a => a.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var purls = record?.Purls?.Where(NotNullOrWhiteSpace).Select(p => p.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var cpes = record?.Cpes?.Where(NotNullOrWhiteSpace).Select(c => c.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var references = record?.References?.Select(r => new VexObservationReference(r.Type, r.Url)).ToImmutableArray() ?? ImmutableArray<VexObservationReference>.Empty;
|
||||
var reconciledFrom = record?.ReconciledFrom?.Where(NotNullOrWhiteSpace).Select(r => r.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var disagreements = record?.Disagreements?.Select(d => new VexObservationDisagreement(d.ProviderId, d.Status, d.Justification, d.Confidence)).ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
var observationRefs = record?.Observations?.Select(o => new VexLinksetObservationRefModel(
|
||||
o.ObservationId,
|
||||
o.ProviderId,
|
||||
o.Status,
|
||||
o.Confidence)).ToImmutableArray() ?? ImmutableArray<VexLinksetObservationRefModel>.Empty;
|
||||
|
||||
return new VexObservationLinkset(aliases, purls, cpes, references, reconciledFrom, disagreements, observationRefs);
|
||||
}
|
||||
|
||||
private static bool NotNullOrWhiteSpace(string? value) => !string.IsNullOrWhiteSpace(value);
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexProviderStore : IVexProviderStore
|
||||
{
|
||||
private readonly IMongoCollection<VexProviderRecord> _collection;
|
||||
|
||||
public MongoVexProviderStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<VexProviderRecord>(VexMongoCollectionNames.Providers);
|
||||
}
|
||||
|
||||
public async ValueTask<VexProvider?> FindAsync(string id, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(id);
|
||||
var filter = Builders<VexProviderRecord>.Filter.Eq(x => x.Id, id.Trim());
|
||||
var record = session is null
|
||||
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
return record?.ToDomain();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyCollection<VexProvider>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
var find = session is null
|
||||
? _collection.Find(FilterDefinition<VexProviderRecord>.Empty)
|
||||
: _collection.Find(session, FilterDefinition<VexProviderRecord>.Empty);
|
||||
var records = await find
|
||||
.Sort(Builders<VexProviderRecord>.Sort.Ascending(x => x.Id))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.ConvertAll(static record => record.ToDomain());
|
||||
}
|
||||
|
||||
public async ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(provider);
|
||||
var record = VexProviderRecord.FromDomain(provider);
|
||||
var filter = Builders<VexProviderRecord>.Filter.Eq(x => x.Id, record.Id);
|
||||
if (session is null)
|
||||
{
|
||||
await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Driver.Core.Clusters;
|
||||
using MongoDB.Driver.GridFS;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Aoc;
|
||||
using StellaOps.Ingestion.Telemetry;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed class MongoVexRawStore : IVexRawStore
|
||||
{
|
||||
private readonly IMongoClient _client;
|
||||
private readonly IMongoCollection<VexRawDocumentRecord> _collection;
|
||||
private readonly GridFSBucket _bucket;
|
||||
private readonly VexMongoStorageOptions _options;
|
||||
private readonly IVexMongoSessionProvider _sessionProvider;
|
||||
private readonly IVexRawWriteGuard _guard;
|
||||
private readonly ILogger<MongoVexRawStore> _logger;
|
||||
private readonly string _connectorVersion;
|
||||
|
||||
public MongoVexRawStore(
|
||||
IMongoClient client,
|
||||
IMongoDatabase database,
|
||||
IOptions<VexMongoStorageOptions> options,
|
||||
IVexMongoSessionProvider sessionProvider,
|
||||
IVexRawWriteGuard guard,
|
||||
ILogger<MongoVexRawStore>? logger = null)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
|
||||
_guard = guard ?? throw new ArgumentNullException(nameof(guard));
|
||||
_logger = logger ?? NullLogger<MongoVexRawStore>.Instance;
|
||||
|
||||
_options = options.Value;
|
||||
Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true);
|
||||
_connectorVersion = typeof(MongoVexRawStore).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||
|
||||
VexMongoMappingRegistry.Register();
|
||||
_collection = database.GetCollection<VexRawDocumentRecord>(VexMongoCollectionNames.Raw);
|
||||
_bucket = new GridFSBucket(database, new GridFSBucketOptions
|
||||
{
|
||||
BucketName = _options.RawBucketName,
|
||||
ReadConcern = database.Settings.ReadConcern,
|
||||
ReadPreference = database.Settings.ReadPreference,
|
||||
WriteConcern = database.Settings.WriteConcern,
|
||||
});
|
||||
}
|
||||
|
||||
public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var guardPayload = VexRawDocumentMapper.ToRawModel(document, _options.DefaultTenant);
|
||||
var tenant = guardPayload.Tenant;
|
||||
var sourceVendor = guardPayload.Source.Vendor;
|
||||
var upstreamId = guardPayload.Upstream.UpstreamId;
|
||||
var contentHash = guardPayload.Upstream.ContentHash;
|
||||
|
||||
using var logScope = _logger.BeginScope(new Dictionary<string, object?>(StringComparer.Ordinal)
|
||||
{
|
||||
["tenant"] = tenant,
|
||||
["source.vendor"] = sourceVendor,
|
||||
["upstream.upstreamId"] = upstreamId,
|
||||
["contentHash"] = contentHash,
|
||||
["providerId"] = document.ProviderId,
|
||||
["digest"] = document.Digest,
|
||||
});
|
||||
|
||||
var transformWatch = Stopwatch.StartNew();
|
||||
using var transformActivity = IngestionTelemetry.StartTransformActivity(
|
||||
tenant,
|
||||
sourceVendor,
|
||||
upstreamId,
|
||||
contentHash,
|
||||
document.Format.ToString(),
|
||||
document.Content.Length);
|
||||
|
||||
try
|
||||
{
|
||||
_guard.EnsureValid(guardPayload);
|
||||
transformActivity?.SetStatus(ActivityStatusCode.Ok);
|
||||
}
|
||||
catch (ExcititorAocGuardException ex)
|
||||
{
|
||||
transformActivity?.SetTag("violationCount", ex.Violations.IsDefaultOrEmpty ? 0 : ex.Violations.Length);
|
||||
transformActivity?.SetTag("code", ex.PrimaryErrorCode);
|
||||
transformActivity?.SetStatus(ActivityStatusCode.Error, ex.PrimaryErrorCode);
|
||||
|
||||
IngestionTelemetry.RecordViolation(tenant, sourceVendor, ex.PrimaryErrorCode);
|
||||
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, IngestionTelemetry.ResultReject);
|
||||
|
||||
_logger.LogWarning(ex, "AOC guard rejected VEX document digest={Digest} provider={ProviderId}", document.Digest, document.ProviderId);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (transformWatch.IsRunning)
|
||||
{
|
||||
transformWatch.Stop();
|
||||
}
|
||||
|
||||
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseTransform, transformWatch.Elapsed);
|
||||
}
|
||||
|
||||
var threshold = _options.GridFsInlineThresholdBytes;
|
||||
var useInline = threshold == 0 || document.Content.Length <= threshold;
|
||||
string? newGridId = null;
|
||||
string? oldGridIdToDelete = null;
|
||||
|
||||
var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone
|
||||
&& !sessionHandle.IsInTransaction;
|
||||
|
||||
var startedTransaction = false;
|
||||
if (supportsTransactions)
|
||||
{
|
||||
try
|
||||
{
|
||||
sessionHandle.StartTransaction();
|
||||
startedTransaction = true;
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
supportsTransactions = false;
|
||||
}
|
||||
}
|
||||
|
||||
var fetchWatch = Stopwatch.StartNew();
|
||||
using var fetchActivity = IngestionTelemetry.StartFetchActivity(
|
||||
tenant,
|
||||
sourceVendor,
|
||||
upstreamId,
|
||||
contentHash,
|
||||
document.SourceUri.ToString());
|
||||
fetchActivity?.SetTag("providerId", document.ProviderId);
|
||||
fetchActivity?.SetTag("format", document.Format.ToString().ToLowerInvariant());
|
||||
|
||||
VexRawDocumentRecord? existing;
|
||||
try
|
||||
{
|
||||
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest);
|
||||
existing = await _collection
|
||||
.Find(sessionHandle, filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
fetchActivity?.SetTag("result", existing is null ? "miss" : "hit");
|
||||
fetchActivity?.SetStatus(ActivityStatusCode.Ok);
|
||||
}
|
||||
catch
|
||||
{
|
||||
fetchActivity?.SetStatus(ActivityStatusCode.Error, "lookup-failed");
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (fetchWatch.IsRunning)
|
||||
{
|
||||
fetchWatch.Stop();
|
||||
}
|
||||
|
||||
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseFetch, fetchWatch.Elapsed);
|
||||
}
|
||||
|
||||
// Append-only: if the digest already exists, skip write
|
||||
if (existing is not null)
|
||||
{
|
||||
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, IngestionTelemetry.ResultNoop);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!useInline)
|
||||
{
|
||||
newGridId = await UploadToGridFsAsync(document, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline);
|
||||
record.GridFsObjectId = useInline ? null : newGridId;
|
||||
|
||||
var writeWatch = Stopwatch.StartNew();
|
||||
using var writeActivity = IngestionTelemetry.StartWriteActivity(
|
||||
tenant,
|
||||
sourceVendor,
|
||||
upstreamId,
|
||||
contentHash,
|
||||
VexMongoCollectionNames.Raw);
|
||||
string? writeResult = null;
|
||||
|
||||
try
|
||||
{
|
||||
await _collection
|
||||
.ReplaceOneAsync(
|
||||
sessionHandle,
|
||||
Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, document.Digest),
|
||||
record,
|
||||
new ReplaceOptions { IsUpsert = true },
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
writeResult = existing is null ? IngestionTelemetry.ResultOk : IngestionTelemetry.ResultNoop;
|
||||
writeActivity?.SetTag("result", writeResult);
|
||||
|
||||
if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId))
|
||||
{
|
||||
if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal))
|
||||
{
|
||||
oldGridIdToDelete = oldGridId;
|
||||
}
|
||||
}
|
||||
|
||||
if (startedTransaction)
|
||||
{
|
||||
await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
writeActivity?.SetStatus(ActivityStatusCode.Ok);
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (startedTransaction && sessionHandle.IsInTransaction)
|
||||
{
|
||||
await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!useInline && !string.IsNullOrWhiteSpace(newGridId))
|
||||
{
|
||||
await DeleteFromGridFsAsync(newGridId, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
writeActivity?.SetStatus(ActivityStatusCode.Error, "write-failed");
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (writeWatch.IsRunning)
|
||||
{
|
||||
writeWatch.Stop();
|
||||
}
|
||||
|
||||
IngestionTelemetry.RecordLatency(tenant, sourceVendor, IngestionTelemetry.PhaseWrite, writeWatch.Elapsed);
|
||||
|
||||
if (!string.IsNullOrEmpty(writeResult))
|
||||
{
|
||||
IngestionTelemetry.RecordWriteAttempt(tenant, sourceVendor, writeResult);
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(oldGridIdToDelete))
|
||||
{
|
||||
await DeleteFromGridFsAsync(oldGridIdToDelete!, sessionHandle, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<VexRawDocument?> FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
throw new ArgumentException("Digest must be provided.", nameof(digest));
|
||||
}
|
||||
|
||||
var trimmed = digest.Trim();
|
||||
var filter = Builders<VexRawDocumentRecord>.Filter.Eq(x => x.Id, trimmed);
|
||||
var record = session is null
|
||||
? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false)
|
||||
: await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (record is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(record.GridFsObjectId))
|
||||
{
|
||||
var handle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, handle, cancellationToken).ConfigureAwait(false);
|
||||
return record.ToDomain(new ReadOnlyMemory<byte>(bytes));
|
||||
}
|
||||
|
||||
return record.ToDomain();
|
||||
}
|
||||
|
||||
private async Task<string?> UploadToGridFsAsync(VexRawDocument document, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
using var stream = new MemoryStream(document.Content.ToArray(), writable: false);
|
||||
var metadata = new BsonDocument
|
||||
{
|
||||
{ "providerId", document.ProviderId },
|
||||
{ "format", document.Format.ToString().ToLowerInvariant() },
|
||||
{ "sourceUri", document.SourceUri.ToString() },
|
||||
{ "retrievedAt", document.RetrievedAt.UtcDateTime },
|
||||
};
|
||||
|
||||
var options = new GridFSUploadOptions { Metadata = metadata };
|
||||
var objectId = await _bucket
|
||||
.UploadFromStreamAsync(document.Digest, stream, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return objectId.ToString();
|
||||
}
|
||||
|
||||
private async Task DeleteFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (GridFSFileNotFoundException)
|
||||
{
|
||||
// file already removed by TTL or manual cleanup
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<byte[]> DownloadFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!ObjectId.TryParse(gridFsObjectId, out var objectId))
|
||||
{
|
||||
return Array.Empty<byte>();
|
||||
}
|
||||
|
||||
return await _bucket.DownloadAsBytesAsync(objectId, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
async ValueTask IVexRawDocumentSink.StoreAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
=> await StoreAsync(document, cancellationToken, session: null).ConfigureAwait(false);
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB record for timeline events.
|
||||
/// </summary>
|
||||
[BsonIgnoreExtraElements]
|
||||
internal sealed class VexTimelineEventRecord
|
||||
{
|
||||
[BsonId]
|
||||
public string Id { get; set; } = default!;
|
||||
|
||||
public string Tenant { get; set; } = default!;
|
||||
|
||||
public string ProviderId { get; set; } = default!;
|
||||
|
||||
public string StreamId { get; set; } = default!;
|
||||
|
||||
public string EventType { get; set; } = default!;
|
||||
|
||||
public string TraceId { get; set; } = default!;
|
||||
|
||||
public string JustificationSummary { get; set; } = string.Empty;
|
||||
|
||||
public string? EvidenceHash { get; set; }
|
||||
|
||||
public string? PayloadHash { get; set; }
|
||||
|
||||
public DateTime CreatedAt { get; set; }
|
||||
= DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
|
||||
|
||||
public Dictionary<string, string> Attributes { get; set; } = new(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB implementation of the timeline event store.
|
||||
/// </summary>
|
||||
internal sealed class MongoVexTimelineEventStore : IVexTimelineEventStore
|
||||
{
|
||||
private readonly IMongoCollection<VexTimelineEventRecord> _collection;
|
||||
|
||||
public MongoVexTimelineEventStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexTimelineEventRecord>(VexMongoCollectionNames.TimelineEvents);
|
||||
}
|
||||
|
||||
public async ValueTask<string> InsertAsync(
|
||||
TimelineEvent evt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(evt);
|
||||
|
||||
var record = ToRecord(evt);
|
||||
|
||||
try
|
||||
{
|
||||
await _collection.InsertOneAsync(record, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return record.Id;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
// Event already exists, return the ID anyway
|
||||
return record.Id;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<int> InsertManyAsync(
|
||||
string tenant,
|
||||
IEnumerable<TimelineEvent> events,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var records = events
|
||||
.Where(e => e is not null && string.Equals(NormalizeTenant(e.Tenant), normalizedTenant, StringComparison.Ordinal))
|
||||
.Select(ToRecord)
|
||||
.ToList();
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var options = new InsertManyOptions { IsOrdered = false };
|
||||
try
|
||||
{
|
||||
await _collection.InsertManyAsync(records, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return records.Count;
|
||||
}
|
||||
catch (MongoBulkWriteException<VexTimelineEventRecord> ex)
|
||||
{
|
||||
var duplicates = ex.WriteErrors?.Count(e => e.Category == ServerErrorCategory.DuplicateKey) ?? 0;
|
||||
return records.Count - duplicates;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByTimeRangeAsync(
|
||||
string tenant,
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var fromUtc = from.UtcDateTime;
|
||||
var toUtc = to.UtcDateTime;
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Gte(r => r.CreatedAt, fromUtc),
|
||||
Builders<VexTimelineEventRecord>.Filter.Lte(r => r.CreatedAt, toUtc));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Ascending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByTraceIdAsync(
|
||||
string tenant,
|
||||
string traceId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedTraceId = traceId?.Trim() ?? throw new ArgumentNullException(nameof(traceId));
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.TraceId, normalizedTraceId));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Ascending(r => r.CreatedAt))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedProvider = providerId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(providerId));
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.ProviderId, normalizedProvider));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByEventTypeAsync(
|
||||
string tenant,
|
||||
string eventType,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedType = eventType?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(eventType));
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.EventType, normalizedType));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> GetRecentAsync(
|
||||
string tenant,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant);
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<TimelineEvent?> GetByIdAsync(
|
||||
string tenant,
|
||||
string eventId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = eventId?.Trim() ?? throw new ArgumentNullException(nameof(eventId));
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Id, normalizedId));
|
||||
|
||||
var record = await _collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return record is null ? null : ToModel(record);
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant);
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountInRangeAsync(
|
||||
string tenant,
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var fromUtc = from.UtcDateTime;
|
||||
var toUtc = to.UtcDateTime;
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Gte(r => r.CreatedAt, fromUtc),
|
||||
Builders<VexTimelineEventRecord>.Filter.Lte(r => r.CreatedAt, toUtc));
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string NormalizeTenant(string tenant)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static VexTimelineEventRecord ToRecord(TimelineEvent evt)
|
||||
{
|
||||
return new VexTimelineEventRecord
|
||||
{
|
||||
Id = evt.EventId,
|
||||
Tenant = evt.Tenant,
|
||||
ProviderId = evt.ProviderId.ToLowerInvariant(),
|
||||
StreamId = evt.StreamId.ToLowerInvariant(),
|
||||
EventType = evt.EventType.ToLowerInvariant(),
|
||||
TraceId = evt.TraceId,
|
||||
JustificationSummary = evt.JustificationSummary,
|
||||
EvidenceHash = evt.EvidenceHash,
|
||||
PayloadHash = evt.PayloadHash,
|
||||
CreatedAt = evt.CreatedAt.UtcDateTime,
|
||||
Attributes = evt.Attributes.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal)
|
||||
};
|
||||
}
|
||||
|
||||
private static TimelineEvent ToModel(VexTimelineEventRecord record)
|
||||
{
|
||||
var attributes = record.Attributes?.ToImmutableDictionary(StringComparer.Ordinal)
|
||||
?? ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
return new TimelineEvent(
|
||||
eventId: record.Id,
|
||||
tenant: record.Tenant,
|
||||
providerId: record.ProviderId,
|
||||
streamId: record.StreamId,
|
||||
eventType: record.EventType,
|
||||
traceId: record.TraceId,
|
||||
justificationSummary: record.JustificationSummary,
|
||||
createdAt: new DateTimeOffset(DateTime.SpecifyKind(record.CreatedAt, DateTimeKind.Utc)),
|
||||
evidenceHash: record.EvidenceHash,
|
||||
payloadHash: record.PayloadHash,
|
||||
attributes: attributes);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Excititor.Storage.Mongo.Tests")]
|
||||
@@ -1,80 +0,0 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public static class VexMongoServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddExcititorMongoStorage(this IServiceCollection services)
|
||||
{
|
||||
services.AddOptions<VexMongoStorageOptions>();
|
||||
|
||||
services.TryAddSingleton<IMongoClient>(static provider =>
|
||||
{
|
||||
var options = provider.GetRequiredService<IOptions<VexMongoStorageOptions>>().Value;
|
||||
Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true);
|
||||
|
||||
var mongoUrl = MongoUrl.Create(options.ConnectionString);
|
||||
var settings = MongoClientSettings.FromUrl(mongoUrl);
|
||||
settings.ReadConcern = ReadConcern.Majority;
|
||||
settings.ReadPreference = ReadPreference.Primary;
|
||||
settings.WriteConcern = WriteConcern.WMajority.With(wTimeout: options.CommandTimeout);
|
||||
settings.RetryReads = true;
|
||||
settings.RetryWrites = true;
|
||||
return new MongoClient(settings);
|
||||
});
|
||||
|
||||
services.TryAddScoped(static provider =>
|
||||
{
|
||||
var options = provider.GetRequiredService<IOptions<VexMongoStorageOptions>>().Value;
|
||||
var client = provider.GetRequiredService<IMongoClient>();
|
||||
|
||||
var settings = new MongoDatabaseSettings
|
||||
{
|
||||
ReadConcern = ReadConcern.Majority,
|
||||
ReadPreference = ReadPreference.PrimaryPreferred,
|
||||
WriteConcern = WriteConcern.WMajority.With(wTimeout: options.CommandTimeout),
|
||||
};
|
||||
|
||||
return client.GetDatabase(options.GetDatabaseName(), settings);
|
||||
});
|
||||
|
||||
services.AddScoped<IVexMongoSessionProvider, VexMongoSessionProvider>();
|
||||
|
||||
services.AddScoped<IVexRawStore, MongoVexRawStore>();
|
||||
services.AddScoped<IVexExportStore, MongoVexExportStore>();
|
||||
services.AddScoped<IVexProviderStore, MongoVexProviderStore>();
|
||||
services.AddScoped<IVexNormalizerRouter, StorageBackedVexNormalizerRouter>();
|
||||
services.AddScoped<IVexConsensusStore, MongoVexConsensusStore>();
|
||||
services.AddScoped<IVexConsensusHoldStore, MongoVexConsensusHoldStore>();
|
||||
services.AddScoped<IVexClaimStore, MongoVexClaimStore>();
|
||||
services.AddScoped<IVexCacheIndex, MongoVexCacheIndex>();
|
||||
services.AddScoped<IVexCacheMaintenance, MongoVexCacheMaintenance>();
|
||||
services.AddScoped<IVexConnectorStateRepository, MongoVexConnectorStateRepository>();
|
||||
services.AddScoped<IAirgapImportStore, MongoAirgapImportStore>();
|
||||
services.AddScoped<VexStatementBackfillService>();
|
||||
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
|
||||
services.AddScoped<IVexObservationStore, MongoVexObservationStore>();
|
||||
services.AddScoped<IVexLinksetStore, MongoVexLinksetStore>();
|
||||
services.AddScoped<IVexLinksetEventPublisher, MongoVexLinksetEventPublisher>();
|
||||
services.AddScoped<VexLinksetDisagreementService>();
|
||||
services.AddScoped<IVexTimelineEventStore, MongoVexTimelineEventStore>();
|
||||
services.AddScoped<IVexTimelineEventEmitter, VexTimelineEventEmitter>();
|
||||
services.AddSingleton<IVexMongoMigration, VexInitialIndexMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexTimelineEventIndexMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexRawSchemaMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexConsensusSignalsMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexConsensusHoldMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexObservationCollectionsMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexRawIdempotencyIndexMigration>();
|
||||
services.AddSingleton<VexMongoMigrationRunner>();
|
||||
services.AddHostedService<VexMongoMigrationHostedService>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Ingestion.Telemetry\StellaOps.Ingestion.Telemetry.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,114 +0,0 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// Normalizer router that resolves providers from Mongo storage before invoking the format-specific normalizer.
|
||||
/// Records telemetry for normalization operations (EXCITITOR-VULN-29-004).
|
||||
/// </summary>
|
||||
public sealed class StorageBackedVexNormalizerRouter : IVexNormalizerRouter
|
||||
{
|
||||
private readonly VexNormalizerRegistry _registry;
|
||||
private readonly IVexProviderStore _providerStore;
|
||||
private readonly IVexMongoSessionProvider _sessionProvider;
|
||||
private readonly ILogger<StorageBackedVexNormalizerRouter> _logger;
|
||||
private readonly IVexNormalizationTelemetryRecorder? _telemetryRecorder;
|
||||
|
||||
public StorageBackedVexNormalizerRouter(
|
||||
IEnumerable<IVexNormalizer> normalizers,
|
||||
IVexProviderStore providerStore,
|
||||
IVexMongoSessionProvider sessionProvider,
|
||||
ILogger<StorageBackedVexNormalizerRouter> logger,
|
||||
IVexNormalizationTelemetryRecorder? telemetryRecorder = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(normalizers);
|
||||
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
|
||||
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_telemetryRecorder = telemetryRecorder;
|
||||
|
||||
_registry = new VexNormalizerRegistry(normalizers.ToImmutableArray());
|
||||
}
|
||||
|
||||
public async ValueTask<VexClaimBatch> NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var normalizer = _registry.Resolve(document);
|
||||
if (normalizer is null)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_logger.LogWarning(
|
||||
"No normalizer registered for VEX document format {Format}. Skipping normalization for {Digest} from provider {ProviderId}.",
|
||||
document.Format,
|
||||
document.Digest,
|
||||
document.ProviderId);
|
||||
|
||||
_telemetryRecorder?.RecordNormalizationError(
|
||||
tenant: null,
|
||||
document.ProviderId,
|
||||
"unsupported_format",
|
||||
$"No normalizer for format {document.Format}");
|
||||
|
||||
return new VexClaimBatch(
|
||||
document,
|
||||
ImmutableArray<VexClaim>.Empty,
|
||||
ImmutableDictionary<string, string>.Empty);
|
||||
}
|
||||
|
||||
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
var provider = await _providerStore.FindAsync(document.ProviderId, cancellationToken, session).ConfigureAwait(false)
|
||||
?? new VexProvider(document.ProviderId, document.ProviderId, VexProviderKind.Vendor);
|
||||
|
||||
try
|
||||
{
|
||||
var batch = await normalizer.NormalizeAsync(document, provider, cancellationToken).ConfigureAwait(false);
|
||||
stopwatch.Stop();
|
||||
|
||||
if (batch.Claims.IsDefaultOrEmpty || batch.Claims.Length == 0)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Normalization produced no claims for document {Digest} from provider {ProviderId}.",
|
||||
document.Digest,
|
||||
document.ProviderId);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"Normalization produced {ClaimCount} claims for document {Digest} from provider {ProviderId} in {Duration}ms.",
|
||||
batch.Claims.Length,
|
||||
document.Digest,
|
||||
document.ProviderId,
|
||||
stopwatch.Elapsed.TotalMilliseconds);
|
||||
}
|
||||
|
||||
return batch;
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Normalization failed for document {Digest} from provider {ProviderId} after {Duration}ms: {Message}",
|
||||
document.Digest,
|
||||
document.ProviderId,
|
||||
stopwatch.Elapsed.TotalMilliseconds,
|
||||
ex.Message);
|
||||
|
||||
_telemetryRecorder?.RecordNormalizationError(
|
||||
tenant: null,
|
||||
document.ProviderId,
|
||||
"normalization_exception",
|
||||
ex.Message);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.RegularExpressions;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Validation;
|
||||
|
||||
/// <summary>
|
||||
/// Validates VEX raw documents against the schema defined in <see cref="Migrations.VexRawSchemaMigration"/>.
|
||||
/// Provides programmatic validation for operators to prove Excititor stores only immutable evidence.
|
||||
/// </summary>
|
||||
public static class VexRawSchemaValidator
|
||||
{
|
||||
private static readonly ImmutableHashSet<string> ValidFormats = ImmutableHashSet.Create(
|
||||
StringComparer.OrdinalIgnoreCase,
|
||||
"csaf", "cyclonedx", "openvex");
|
||||
|
||||
private static readonly ImmutableHashSet<BsonType> ValidContentTypes = ImmutableHashSet.Create(
|
||||
BsonType.Binary, BsonType.String);
|
||||
|
||||
private static readonly ImmutableHashSet<BsonType> ValidGridFsTypes = ImmutableHashSet.Create(
|
||||
BsonType.ObjectId, BsonType.Null, BsonType.String);
|
||||
|
||||
/// <summary>
|
||||
/// Validates a VEX raw document against the schema requirements.
|
||||
/// </summary>
|
||||
/// <param name="document">The document to validate.</param>
|
||||
/// <returns>Validation result with any violations found.</returns>
|
||||
public static VexRawValidationResult Validate(BsonDocument document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var violations = new List<VexRawSchemaViolation>();
|
||||
|
||||
// Required fields
|
||||
ValidateRequired(document, "_id", violations);
|
||||
ValidateRequired(document, "providerId", violations);
|
||||
ValidateRequired(document, "format", violations);
|
||||
ValidateRequired(document, "sourceUri", violations);
|
||||
ValidateRequired(document, "retrievedAt", violations);
|
||||
ValidateRequired(document, "digest", violations);
|
||||
|
||||
// Field types and constraints
|
||||
ValidateStringField(document, "_id", minLength: 1, violations);
|
||||
ValidateStringField(document, "providerId", minLength: 1, violations);
|
||||
ValidateFormatEnum(document, violations);
|
||||
ValidateStringField(document, "sourceUri", minLength: 1, violations);
|
||||
ValidateDateField(document, "retrievedAt", violations);
|
||||
ValidateStringField(document, "digest", minLength: 32, violations);
|
||||
|
||||
// Optional fields with type constraints
|
||||
if (document.Contains("content"))
|
||||
{
|
||||
ValidateContentField(document, violations);
|
||||
}
|
||||
|
||||
if (document.Contains("gridFsObjectId"))
|
||||
{
|
||||
ValidateGridFsObjectIdField(document, violations);
|
||||
}
|
||||
|
||||
if (document.Contains("metadata"))
|
||||
{
|
||||
ValidateMetadataField(document, violations);
|
||||
}
|
||||
|
||||
return new VexRawValidationResult(
|
||||
document.GetValue("_id", BsonNull.Value).ToString() ?? "<unknown>",
|
||||
violations.Count == 0,
|
||||
violations.ToImmutableArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates multiple documents and returns aggregated results.
|
||||
/// </summary>
|
||||
public static VexRawBatchValidationResult ValidateBatch(IEnumerable<BsonDocument> documents)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(documents);
|
||||
|
||||
var results = new List<VexRawValidationResult>();
|
||||
foreach (var doc in documents)
|
||||
{
|
||||
results.Add(Validate(doc));
|
||||
}
|
||||
|
||||
var valid = results.Count(r => r.IsValid);
|
||||
var invalid = results.Count(r => !r.IsValid);
|
||||
|
||||
return new VexRawBatchValidationResult(
|
||||
results.Count,
|
||||
valid,
|
||||
invalid,
|
||||
results.Where(r => !r.IsValid).ToImmutableArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the MongoDB JSON Schema document for offline validation.
|
||||
/// </summary>
|
||||
public static BsonDocument GetJsonSchema()
|
||||
{
|
||||
var properties = new BsonDocument
|
||||
{
|
||||
{ "_id", new BsonDocument { { "bsonType", "string" }, { "description", "Content digest serving as immutable key" } } },
|
||||
{ "providerId", new BsonDocument { { "bsonType", "string" }, { "minLength", 1 }, { "description", "VEX provider identifier" } } },
|
||||
{ "format", new BsonDocument
|
||||
{
|
||||
{ "bsonType", "string" },
|
||||
{ "enum", new BsonArray { "csaf", "cyclonedx", "openvex" } },
|
||||
{ "description", "VEX document format" }
|
||||
}
|
||||
},
|
||||
{ "sourceUri", new BsonDocument { { "bsonType", "string" }, { "minLength", 1 }, { "description", "Original source URI" } } },
|
||||
{ "retrievedAt", new BsonDocument { { "bsonType", "date" }, { "description", "Timestamp when document was fetched" } } },
|
||||
{ "digest", new BsonDocument { { "bsonType", "string" }, { "minLength", 32 }, { "description", "Content hash (SHA-256 hex)" } } },
|
||||
{ "content", new BsonDocument
|
||||
{
|
||||
{ "bsonType", new BsonArray { "binData", "string" } },
|
||||
{ "description", "Raw document content (binary or base64 string)" }
|
||||
}
|
||||
},
|
||||
{ "gridFsObjectId", new BsonDocument
|
||||
{
|
||||
{ "bsonType", new BsonArray { "objectId", "null", "string" } },
|
||||
{ "description", "GridFS reference for large documents" }
|
||||
}
|
||||
},
|
||||
{ "metadata", new BsonDocument
|
||||
{
|
||||
{ "bsonType", "object" },
|
||||
{ "additionalProperties", true },
|
||||
{ "description", "Provider-specific metadata (string values only)" }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return new BsonDocument
|
||||
{
|
||||
{
|
||||
"$jsonSchema",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "bsonType", "object" },
|
||||
{ "title", "VEX Raw Document Schema" },
|
||||
{ "description", "Schema for immutable VEX evidence storage. Documents are content-addressed and must not be modified after insertion." },
|
||||
{ "required", new BsonArray { "_id", "providerId", "format", "sourceUri", "retrievedAt", "digest" } },
|
||||
{ "properties", properties },
|
||||
{ "additionalProperties", true }
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the schema as a JSON string for operator documentation.
|
||||
/// </summary>
|
||||
public static string GetJsonSchemaAsJson()
|
||||
{
|
||||
return GetJsonSchema().ToJson(new MongoDB.Bson.IO.JsonWriterSettings { Indent = true });
|
||||
}
|
||||
|
||||
private static void ValidateRequired(BsonDocument doc, string field, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
if (!doc.Contains(field) || doc[field].IsBsonNull)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation(field, $"Required field '{field}' is missing or null"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateStringField(BsonDocument doc, string field, int minLength, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
if (!doc.Contains(field))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var value = doc[field];
|
||||
if (value.IsBsonNull)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!value.IsString)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation(field, $"Field '{field}' must be a string, got {value.BsonType}"));
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.AsString.Length < minLength)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation(field, $"Field '{field}' must have minimum length {minLength}, got {value.AsString.Length}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateFormatEnum(BsonDocument doc, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
if (!doc.Contains("format"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var value = doc["format"];
|
||||
if (value.IsBsonNull || !value.IsString)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ValidFormats.Contains(value.AsString))
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation("format", $"Field 'format' must be one of [{string.Join(", ", ValidFormats)}], got '{value.AsString}'"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateDateField(BsonDocument doc, string field, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
if (!doc.Contains(field))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var value = doc[field];
|
||||
if (value.IsBsonNull)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.BsonType != BsonType.DateTime)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation(field, $"Field '{field}' must be a date, got {value.BsonType}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateContentField(BsonDocument doc, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
var value = doc["content"];
|
||||
if (value.IsBsonNull)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ValidContentTypes.Contains(value.BsonType))
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation("content", $"Field 'content' must be binary or string, got {value.BsonType}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateGridFsObjectIdField(BsonDocument doc, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
var value = doc["gridFsObjectId"];
|
||||
if (!ValidGridFsTypes.Contains(value.BsonType))
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation("gridFsObjectId", $"Field 'gridFsObjectId' must be objectId, null, or string, got {value.BsonType}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateMetadataField(BsonDocument doc, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
var value = doc["metadata"];
|
||||
if (value.IsBsonNull)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.BsonType != BsonType.Document)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation("metadata", $"Field 'metadata' must be an object, got {value.BsonType}"));
|
||||
return;
|
||||
}
|
||||
|
||||
var metadata = value.AsBsonDocument;
|
||||
foreach (var element in metadata)
|
||||
{
|
||||
if (!element.Value.IsString && !element.Value.IsBsonNull)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation($"metadata.{element.Name}", $"Metadata field '{element.Name}' must be a string, got {element.Value.BsonType}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a schema violation found during validation.
|
||||
/// </summary>
|
||||
public sealed record VexRawSchemaViolation(string Field, string Message);
|
||||
|
||||
/// <summary>
|
||||
/// Result of validating a single VEX raw document.
|
||||
/// </summary>
|
||||
public sealed record VexRawValidationResult(
|
||||
string DocumentId,
|
||||
bool IsValid,
|
||||
ImmutableArray<VexRawSchemaViolation> Violations);
|
||||
|
||||
/// <summary>
|
||||
/// Result of validating a batch of VEX raw documents.
|
||||
/// </summary>
|
||||
public sealed record VexRawBatchValidationResult(
|
||||
int TotalCount,
|
||||
int ValidCount,
|
||||
int InvalidCount,
|
||||
ImmutableArray<VexRawValidationResult> InvalidDocuments);
|
||||
@@ -1,63 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
[BsonIgnoreExtraElements]
|
||||
internal sealed class VexAttestationLinkRecord
|
||||
{
|
||||
[BsonId]
|
||||
public string AttestationId { get; set; } = default!;
|
||||
|
||||
public string SupplierId { get; set; } = default!;
|
||||
|
||||
public string ObservationId { get; set; } = default!;
|
||||
|
||||
public string LinksetId { get; set; } = default!;
|
||||
|
||||
public string VulnerabilityId { get; set; } = default!;
|
||||
|
||||
public string ProductKey { get; set; } = default!;
|
||||
|
||||
public string? JustificationSummary { get; set; }
|
||||
= null;
|
||||
|
||||
public DateTime IssuedAt { get; set; }
|
||||
= DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
|
||||
|
||||
public Dictionary<string, string> Metadata { get; set; } = new(StringComparer.Ordinal);
|
||||
|
||||
public static VexAttestationLinkRecord FromDomain(VexAttestationPayload payload)
|
||||
=> new()
|
||||
{
|
||||
AttestationId = payload.AttestationId,
|
||||
SupplierId = payload.SupplierId,
|
||||
ObservationId = payload.ObservationId,
|
||||
LinksetId = payload.LinksetId,
|
||||
VulnerabilityId = payload.VulnerabilityId,
|
||||
ProductKey = payload.ProductKey,
|
||||
JustificationSummary = payload.JustificationSummary,
|
||||
IssuedAt = payload.IssuedAt.UtcDateTime,
|
||||
Metadata = payload.Metadata.ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.Ordinal),
|
||||
};
|
||||
|
||||
public VexAttestationPayload ToDomain()
|
||||
{
|
||||
var metadata = (Metadata ?? new Dictionary<string, string>(StringComparer.Ordinal))
|
||||
.ToImmutableDictionary(StringComparer.Ordinal);
|
||||
|
||||
return new VexAttestationPayload(
|
||||
AttestationId,
|
||||
SupplierId,
|
||||
ObservationId,
|
||||
LinksetId,
|
||||
VulnerabilityId,
|
||||
ProductKey,
|
||||
JustificationSummary,
|
||||
new DateTimeOffset(DateTime.SpecifyKind(IssuedAt, DateTimeKind.Utc)),
|
||||
metadata);
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
using System.Threading;
|
||||
using MongoDB.Bson.Serialization;
|
||||
using MongoDB.Bson.Serialization.Serializers;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public static class VexMongoMappingRegistry
|
||||
{
|
||||
private static int _initialized;
|
||||
|
||||
public static void Register()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _initialized, 1) == 1)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
BsonSerializer.RegisterSerializer(typeof(byte[]), new ByteArraySerializer());
|
||||
}
|
||||
catch
|
||||
{
|
||||
// serializer already registered – safe to ignore
|
||||
}
|
||||
|
||||
RegisterClassMaps();
|
||||
}
|
||||
|
||||
private static void RegisterClassMaps()
|
||||
{
|
||||
RegisterClassMap<VexProviderRecord>();
|
||||
RegisterClassMap<VexProviderDiscoveryDocument>();
|
||||
RegisterClassMap<VexProviderTrustDocument>();
|
||||
RegisterClassMap<VexCosignTrustDocument>();
|
||||
RegisterClassMap<VexConsensusRecord>();
|
||||
RegisterClassMap<VexProductDocument>();
|
||||
RegisterClassMap<VexConsensusSourceDocument>();
|
||||
RegisterClassMap<VexConsensusConflictDocument>();
|
||||
RegisterClassMap<VexConfidenceDocument>();
|
||||
RegisterClassMap<VexSignalDocument>();
|
||||
RegisterClassMap<VexSeveritySignalDocument>();
|
||||
RegisterClassMap<VexClaimDocumentRecord>();
|
||||
RegisterClassMap<VexSignatureMetadataDocument>();
|
||||
RegisterClassMap<VexStatementRecord>();
|
||||
RegisterClassMap<VexCacheEntryRecord>();
|
||||
RegisterClassMap<VexConnectorStateDocument>();
|
||||
RegisterClassMap<VexConsensusHoldRecord>();
|
||||
RegisterClassMap<AirgapImportRecord>();
|
||||
RegisterClassMap<VexTimelineEventRecord>();
|
||||
}
|
||||
|
||||
private static void RegisterClassMap<TDocument>()
|
||||
where TDocument : class
|
||||
{
|
||||
if (BsonClassMap.IsClassMapRegistered(typeof(TDocument)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
BsonClassMap.RegisterClassMap<TDocument>(classMap =>
|
||||
{
|
||||
classMap.AutoMap();
|
||||
classMap.SetIgnoreExtraElements(true);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public static class VexMongoCollectionNames
|
||||
{
|
||||
public const string Migrations = "vex.migrations";
|
||||
public const string Providers = "vex.providers";
|
||||
public const string Raw = "vex.raw";
|
||||
public const string Statements = "vex.statements";
|
||||
public const string Claims = Statements;
|
||||
public const string Consensus = "vex.consensus";
|
||||
public const string Exports = "vex.exports";
|
||||
public const string Cache = "vex.cache";
|
||||
public const string ConnectorState = "vex.connector_state";
|
||||
public const string ConsensusHolds = "vex.consensus_holds";
|
||||
public const string Attestations = "vex.attestations";
|
||||
public const string Observations = "vex.observations";
|
||||
public const string Linksets = "vex.linksets";
|
||||
public const string LinksetEvents = "vex.linkset_events";
|
||||
public const string AirgapImports = "vex.airgap_imports";
|
||||
public const string TimelineEvents = "vex.timeline_events";
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,120 +0,0 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public interface IVexMongoSessionProvider : IAsyncDisposable
|
||||
{
|
||||
ValueTask<IClientSessionHandle> StartSessionAsync(CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
internal sealed class VexMongoSessionProvider : IVexMongoSessionProvider
|
||||
{
|
||||
private readonly IMongoClient _client;
|
||||
private readonly VexMongoStorageOptions _options;
|
||||
private readonly object _gate = new();
|
||||
private Task<IClientSessionHandle>? _sessionTask;
|
||||
private IClientSessionHandle? _session;
|
||||
private bool _disposed;
|
||||
|
||||
public VexMongoSessionProvider(IMongoClient client, IOptions<VexMongoStorageOptions> options)
|
||||
{
|
||||
_client = client ?? throw new ArgumentNullException(nameof(client));
|
||||
if (options is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
_options = options.Value;
|
||||
}
|
||||
|
||||
public async ValueTask<IClientSessionHandle> StartSessionAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
|
||||
var existing = Volatile.Read(ref _session);
|
||||
if (existing is not null)
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
Task<IClientSessionHandle> startTask;
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
if (_session is { } current)
|
||||
{
|
||||
return current;
|
||||
}
|
||||
|
||||
_sessionTask ??= StartSessionInternalAsync(cancellationToken);
|
||||
startTask = _sessionTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var handle = await startTask.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (_session is null)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_session is null)
|
||||
{
|
||||
_session = handle;
|
||||
_sessionTask = Task.FromResult(handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return handle;
|
||||
}
|
||||
catch
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (ReferenceEquals(_sessionTask, startTask))
|
||||
{
|
||||
_sessionTask = null;
|
||||
}
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private Task<IClientSessionHandle> StartSessionInternalAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var sessionOptions = new ClientSessionOptions
|
||||
{
|
||||
CausalConsistency = true,
|
||||
DefaultTransactionOptions = new TransactionOptions(
|
||||
readPreference: ReadPreference.Primary,
|
||||
readConcern: ReadConcern.Majority,
|
||||
writeConcern: WriteConcern.WMajority.With(wTimeout: _options.CommandTimeout))
|
||||
};
|
||||
|
||||
return _client.StartSessionAsync(sessionOptions, cancellationToken);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
|
||||
IClientSessionHandle? handle;
|
||||
lock (_gate)
|
||||
{
|
||||
handle = _session;
|
||||
_session = null;
|
||||
_sessionTask = null;
|
||||
}
|
||||
|
||||
handle?.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration controlling Mongo-backed storage for Excititor repositories.
|
||||
/// </summary>
|
||||
public sealed class VexMongoStorageOptions : IValidatableObject
|
||||
{
|
||||
private const int DefaultInlineThreshold = 256 * 1024;
|
||||
private static readonly TimeSpan DefaultCacheTtl = TimeSpan.FromHours(12);
|
||||
private static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB connection string for Excititor storage.
|
||||
/// </summary>
|
||||
public string ConnectionString { get; set; } = "mongodb://localhost:27017";
|
||||
|
||||
/// <summary>
|
||||
/// Overrides the database name extracted from <see cref="ConnectionString"/>.
|
||||
/// </summary>
|
||||
public string? DatabaseName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to write operations to ensure majority acknowledgement completes promptly.
|
||||
/// </summary>
|
||||
public TimeSpan CommandTimeout { get; set; } = DefaultCommandTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Name of the GridFS bucket used for raw VEX payloads that exceed <see cref="GridFsInlineThresholdBytes"/>.
|
||||
/// </summary>
|
||||
public string RawBucketName { get; set; } = "vex.raw";
|
||||
|
||||
/// <summary>
|
||||
/// Inline raw document payloads smaller than this threshold; larger payloads are stored in GridFS.
|
||||
/// </summary>
|
||||
public int GridFsInlineThresholdBytes { get; set; } = DefaultInlineThreshold;
|
||||
|
||||
/// <summary>
|
||||
/// Default TTL applied to export cache entries (absolute expiration).
|
||||
/// </summary>
|
||||
public TimeSpan ExportCacheTtl { get; set; } = DefaultCacheTtl;
|
||||
|
||||
/// <summary>
|
||||
/// Tenant identifier associated with raw VEX documents persisted by this storage instance.
|
||||
/// </summary>
|
||||
public string DefaultTenant { get; set; } = "tenant-default";
|
||||
|
||||
/// <summary>
|
||||
/// Resolve the Mongo database name using the explicit override or connection string.
|
||||
/// </summary>
|
||||
public string GetDatabaseName()
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(DatabaseName))
|
||||
{
|
||||
return DatabaseName.Trim();
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ConnectionString))
|
||||
{
|
||||
var url = MongoUrl.Create(ConnectionString);
|
||||
if (!string.IsNullOrWhiteSpace(url.DatabaseName))
|
||||
{
|
||||
return url.DatabaseName;
|
||||
}
|
||||
}
|
||||
|
||||
return "excititor";
|
||||
}
|
||||
|
||||
public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(ConnectionString))
|
||||
{
|
||||
yield return new ValidationResult("Mongo connection string must be provided.", new[] { nameof(ConnectionString) });
|
||||
}
|
||||
|
||||
if (CommandTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
yield return new ValidationResult("Command timeout must be greater than zero.", new[] { nameof(CommandTimeout) });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(RawBucketName))
|
||||
{
|
||||
yield return new ValidationResult("Raw bucket name must be provided.", new[] { nameof(RawBucketName) });
|
||||
}
|
||||
|
||||
if (GridFsInlineThresholdBytes < 0)
|
||||
{
|
||||
yield return new ValidationResult("GridFS inline threshold must be non-negative.", new[] { nameof(GridFsInlineThresholdBytes) });
|
||||
}
|
||||
|
||||
if (ExportCacheTtl <= TimeSpan.Zero)
|
||||
{
|
||||
yield return new ValidationResult("Export cache TTL must be greater than zero.", new[] { nameof(ExportCacheTtl) });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(DefaultTenant))
|
||||
{
|
||||
yield return new ValidationResult("Default tenant must be provided.", new[] { nameof(DefaultTenant) });
|
||||
}
|
||||
|
||||
_ = GetDatabaseName();
|
||||
}
|
||||
}
|
||||
@@ -1,194 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Excititor.Core;
|
||||
using RawVexDocumentModel = StellaOps.Concelier.RawModels.VexRawDocument;
|
||||
using RawSourceMetadata = StellaOps.Concelier.RawModels.RawSourceMetadata;
|
||||
using RawUpstreamMetadata = StellaOps.Concelier.RawModels.RawUpstreamMetadata;
|
||||
using RawContentMetadata = StellaOps.Concelier.RawModels.RawContent;
|
||||
using RawSignatureMetadata = StellaOps.Concelier.RawModels.RawSignatureMetadata;
|
||||
using RawLinkset = StellaOps.Concelier.RawModels.RawLinkset;
|
||||
using RawReference = StellaOps.Concelier.RawModels.RawReference;
|
||||
using VexStatementSummaryModel = StellaOps.Concelier.RawModels.VexStatementSummary;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// Converts Excititor domain VEX documents into Aggregation-Only Contract raw payloads.
|
||||
/// </summary>
|
||||
public static class VexRawDocumentMapper
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public static RawVexDocumentModel ToRawModel(VexRawDocument document, string defaultTenant)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var metadata = document.Metadata ?? ImmutableDictionary<string, string>.Empty;
|
||||
var tenant = ResolveTenant(metadata, defaultTenant);
|
||||
var source = CreateSourceMetadata(document, metadata);
|
||||
var upstream = CreateUpstreamMetadata(document, metadata);
|
||||
var content = CreateContent(document, metadata);
|
||||
var linkset = CreateLinkset();
|
||||
ImmutableArray<VexStatementSummaryModel>? statements = null;
|
||||
return new RawVexDocumentModel(tenant, source, upstream, content, linkset, statements);
|
||||
}
|
||||
|
||||
private static string ResolveTenant(ImmutableDictionary<string, string> metadata, string fallback)
|
||||
{
|
||||
var tenant = TryMetadata(metadata, "tenant", "aoc.tenant");
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
return (fallback ?? "tenant-default").Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static RawSourceMetadata CreateSourceMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
var vendor = TryMetadata(metadata, "source.vendor", "connector.vendor") ?? ExtractVendor(document.ProviderId);
|
||||
var connector = TryMetadata(metadata, "source.connector") ?? document.ProviderId;
|
||||
var version = TryMetadata(metadata, "source.connector_version", "connector.version") ?? GetAssemblyVersion();
|
||||
var stream = TryMetadata(metadata, "source.stream", "connector.stream") ?? document.Format.ToString().ToLowerInvariant();
|
||||
return new RawSourceMetadata(vendor, connector, version, stream);
|
||||
}
|
||||
|
||||
private static string GetAssemblyVersion()
|
||||
=> typeof(VexRawDocumentMapper).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||
|
||||
private static RawUpstreamMetadata CreateUpstreamMetadata(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
var upstreamId = TryMetadata(
|
||||
metadata,
|
||||
"upstream.id",
|
||||
"aoc.upstream_id",
|
||||
"vulnerability.id",
|
||||
"advisory.id",
|
||||
"msrc.vulnerabilityId",
|
||||
"msrc.advisoryId",
|
||||
"oracle.csaf.entryId",
|
||||
"ubuntu.advisoryId",
|
||||
"cisco.csaf.documentId",
|
||||
"rancher.vex.id") ?? document.SourceUri.ToString();
|
||||
|
||||
var documentVersion = TryMetadata(
|
||||
metadata,
|
||||
"upstream.version",
|
||||
"aoc.document_version",
|
||||
"msrc.lastModified",
|
||||
"msrc.releaseDate",
|
||||
"oracle.csaf.revision",
|
||||
"ubuntu.version",
|
||||
"ubuntu.lastModified",
|
||||
"cisco.csaf.revision") ?? document.RetrievedAt.ToString("O");
|
||||
|
||||
var signature = CreateSignatureMetadata(metadata);
|
||||
return new RawUpstreamMetadata(
|
||||
upstreamId,
|
||||
documentVersion,
|
||||
document.RetrievedAt,
|
||||
document.Digest,
|
||||
signature,
|
||||
metadata);
|
||||
}
|
||||
|
||||
private static RawSignatureMetadata CreateSignatureMetadata(ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
if (!TryBool(metadata, out var present, "signature.present", "aoc.signature.present"))
|
||||
{
|
||||
return new RawSignatureMetadata(false);
|
||||
}
|
||||
|
||||
if (!present)
|
||||
{
|
||||
return new RawSignatureMetadata(false);
|
||||
}
|
||||
|
||||
var format = TryMetadata(metadata, "signature.format", "aoc.signature.format");
|
||||
var keyId = TryMetadata(metadata, "signature.key_id", "signature.keyId", "aoc.signature.key_id");
|
||||
var signature = TryMetadata(metadata, "signature.sig", "signature.signature", "aoc.signature.sig");
|
||||
var digest = TryMetadata(metadata, "signature.digest", "aoc.signature.digest");
|
||||
var certificate = TryMetadata(metadata, "signature.certificate", "aoc.signature.certificate");
|
||||
return new RawSignatureMetadata(true, format, keyId, signature, certificate, digest);
|
||||
}
|
||||
|
||||
private static RawContentMetadata CreateContent(VexRawDocument document, ImmutableDictionary<string, string> metadata)
|
||||
{
|
||||
if (document.Content.IsEmpty)
|
||||
{
|
||||
throw new InvalidOperationException("Raw VEX document content cannot be empty when enforcing AOC guard.");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var payload = JsonDocument.Parse(document.Content.ToArray());
|
||||
var raw = payload.RootElement.Clone();
|
||||
var specVersion = TryMetadata(metadata, "content.spec_version", "csaf.version", "openvex.version");
|
||||
var encoding = TryMetadata(metadata, "content.encoding");
|
||||
return new RawContentMetadata(
|
||||
document.Format.ToString(),
|
||||
specVersion,
|
||||
raw,
|
||||
encoding);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidOperationException("Raw VEX document payload must be valid JSON for AOC guard enforcement.", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static RawLinkset CreateLinkset()
|
||||
=> new()
|
||||
{
|
||||
Aliases = ImmutableArray<string>.Empty,
|
||||
PackageUrls = ImmutableArray<string>.Empty,
|
||||
Cpes = ImmutableArray<string>.Empty,
|
||||
References = ImmutableArray<RawReference>.Empty,
|
||||
ReconciledFrom = ImmutableArray<string>.Empty,
|
||||
Notes = ImmutableDictionary<string, string>.Empty,
|
||||
};
|
||||
|
||||
private static string? TryMetadata(ImmutableDictionary<string, string> metadata, params string[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool TryBool(ImmutableDictionary<string, string> metadata, out bool value, params string[] keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var text) && bool.TryParse(text, out value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string ExtractVendor(string providerId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(providerId))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var trimmed = providerId.Trim();
|
||||
var separatorIndex = trimmed.LastIndexOfAny(new[] { ':', '.' });
|
||||
if (separatorIndex >= 0 && separatorIndex < trimmed.Length - 1)
|
||||
{
|
||||
return trimmed[(separatorIndex + 1)..];
|
||||
}
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
public sealed record VexStatementBackfillRequest(
|
||||
DateTimeOffset? RetrievedSince = null,
|
||||
bool Force = false,
|
||||
int BatchSize = 100,
|
||||
int? MaxDocuments = null);
|
||||
|
||||
public sealed record VexStatementBackfillResult(
|
||||
int DocumentsEvaluated,
|
||||
int DocumentsBackfilled,
|
||||
int ClaimsWritten,
|
||||
int SkippedExisting,
|
||||
int NormalizationFailures);
|
||||
|
||||
public sealed class VexStatementBackfillService
|
||||
{
|
||||
private readonly IVexRawStore _rawStore;
|
||||
private readonly IVexNormalizerRouter _normalizerRouter;
|
||||
private readonly IVexClaimStore _claimStore;
|
||||
private readonly IVexMongoSessionProvider _sessionProvider;
|
||||
private readonly IMongoCollection<VexRawDocumentRecord> _rawCollection;
|
||||
private readonly IMongoCollection<VexStatementRecord> _statementCollection;
|
||||
private readonly ILogger<VexStatementBackfillService> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public VexStatementBackfillService(
|
||||
IMongoDatabase database,
|
||||
IVexRawStore rawStore,
|
||||
IVexNormalizerRouter normalizerRouter,
|
||||
IVexClaimStore claimStore,
|
||||
IVexMongoSessionProvider sessionProvider,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<VexStatementBackfillService> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
|
||||
_normalizerRouter = normalizerRouter ?? throw new ArgumentNullException(nameof(normalizerRouter));
|
||||
_claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
|
||||
_sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
VexMongoMappingRegistry.Register();
|
||||
_rawCollection = database.GetCollection<VexRawDocumentRecord>(VexMongoCollectionNames.Raw);
|
||||
_statementCollection = database.GetCollection<VexStatementRecord>(VexMongoCollectionNames.Statements);
|
||||
}
|
||||
|
||||
public async Task<VexStatementBackfillResult> RunAsync(VexStatementBackfillRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
if (request.BatchSize < 1)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(request.BatchSize), "Batch size must be at least 1.");
|
||||
}
|
||||
|
||||
if (request.MaxDocuments is { } max && max <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(request.MaxDocuments), "Max documents must be positive when specified.");
|
||||
}
|
||||
|
||||
var evaluated = 0;
|
||||
var backfilled = 0;
|
||||
var claimsWritten = 0;
|
||||
var skipped = 0;
|
||||
var failures = 0;
|
||||
|
||||
var filter = request.RetrievedSince is { } since
|
||||
? Builders<VexRawDocumentRecord>.Filter.Gte(x => x.RetrievedAt, since.UtcDateTime)
|
||||
: FilterDefinition<VexRawDocumentRecord>.Empty;
|
||||
|
||||
var findOptions = new FindOptions<VexRawDocumentRecord>
|
||||
{
|
||||
Sort = Builders<VexRawDocumentRecord>.Sort.Ascending(x => x.RetrievedAt),
|
||||
BatchSize = request.BatchSize,
|
||||
};
|
||||
|
||||
if (request.MaxDocuments is { } limit)
|
||||
{
|
||||
findOptions.Limit = limit;
|
||||
}
|
||||
|
||||
using var cursor = await _rawCollection.FindAsync(filter, findOptions, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var record in cursor.Current)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
evaluated++;
|
||||
|
||||
if (!request.Force && await StatementExistsAsync(record.Digest, session, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var rawDocument = await _rawStore.FindByDigestAsync(record.Digest, cancellationToken, session).ConfigureAwait(false);
|
||||
if (rawDocument is null)
|
||||
{
|
||||
failures++;
|
||||
_logger.LogWarning("Backfill skipped missing raw document {Digest}.", record.Digest);
|
||||
continue;
|
||||
}
|
||||
|
||||
VexClaimBatch batch;
|
||||
try
|
||||
{
|
||||
batch = await _normalizerRouter.NormalizeAsync(rawDocument, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
failures++;
|
||||
_logger.LogError(ex, "Failed to normalize raw document {Digest} during statement backfill.", record.Digest);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (batch.Claims.IsDefaultOrEmpty || batch.Claims.Length == 0)
|
||||
{
|
||||
failures++;
|
||||
_logger.LogWarning("Backfill produced no claims for {Digest}; skipping.", record.Digest);
|
||||
continue;
|
||||
}
|
||||
|
||||
var claims = batch.Claims.AsEnumerable();
|
||||
var observedAt = rawDocument.RetrievedAt == default
|
||||
? _timeProvider.GetUtcNow()
|
||||
: rawDocument.RetrievedAt;
|
||||
|
||||
await _claimStore.AppendAsync(claims, observedAt, cancellationToken, session).ConfigureAwait(false);
|
||||
|
||||
backfilled++;
|
||||
claimsWritten += batch.Claims.Length;
|
||||
}
|
||||
}
|
||||
|
||||
var result = new VexStatementBackfillResult(evaluated, backfilled, claimsWritten, skipped, failures);
|
||||
_logger.LogInformation(
|
||||
"Statement backfill completed: evaluated {Evaluated} documents, backfilled {Backfilled}, wrote {Claims} claims, skipped {Skipped}, failures {Failures}.",
|
||||
result.DocumentsEvaluated,
|
||||
result.DocumentsBackfilled,
|
||||
result.ClaimsWritten,
|
||||
result.SkippedExisting,
|
||||
result.NormalizationFailures);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private Task<bool> StatementExistsAsync(string digest, IClientSessionHandle session, CancellationToken cancellationToken)
|
||||
{
|
||||
var filter = Builders<VexStatementRecord>.Filter.Eq(x => x.Document.Digest, digest);
|
||||
var find = session is null
|
||||
? _statementCollection.Find(filter)
|
||||
: _statementCollection.Find(session, filter);
|
||||
return find.Limit(1).AnyAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -1,169 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IVexTimelineEventEmitter"/> that persists events to MongoDB.
|
||||
/// </summary>
|
||||
internal sealed class VexTimelineEventEmitter : IVexTimelineEventEmitter
|
||||
{
|
||||
private readonly IVexTimelineEventStore _store;
|
||||
private readonly ILogger<VexTimelineEventEmitter> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public VexTimelineEventEmitter(
|
||||
IVexTimelineEventStore store,
|
||||
ILogger<VexTimelineEventEmitter> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async ValueTask EmitObservationIngestAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
string streamId,
|
||||
string traceId,
|
||||
string observationId,
|
||||
string evidenceHash,
|
||||
string justificationSummary,
|
||||
ImmutableDictionary<string, string>? attributes = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var eventAttributes = (attributes ?? ImmutableDictionary<string, string>.Empty)
|
||||
.SetItem(VexTimelineEventAttributes.ObservationId, observationId);
|
||||
|
||||
var evt = new TimelineEvent(
|
||||
eventId: GenerateEventId(tenant, providerId, VexTimelineEventTypes.ObservationIngested),
|
||||
tenant: tenant,
|
||||
providerId: providerId,
|
||||
streamId: streamId,
|
||||
eventType: VexTimelineEventTypes.ObservationIngested,
|
||||
traceId: traceId,
|
||||
justificationSummary: justificationSummary,
|
||||
createdAt: _timeProvider.GetUtcNow(),
|
||||
evidenceHash: evidenceHash,
|
||||
payloadHash: null,
|
||||
attributes: eventAttributes);
|
||||
|
||||
await EmitAsync(evt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask EmitLinksetUpdateAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
string streamId,
|
||||
string traceId,
|
||||
string linksetId,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
string payloadHash,
|
||||
string justificationSummary,
|
||||
ImmutableDictionary<string, string>? attributes = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var eventAttributes = (attributes ?? ImmutableDictionary<string, string>.Empty)
|
||||
.SetItem(VexTimelineEventAttributes.LinksetId, linksetId)
|
||||
.SetItem(VexTimelineEventAttributes.VulnerabilityId, vulnerabilityId)
|
||||
.SetItem(VexTimelineEventAttributes.ProductKey, productKey);
|
||||
|
||||
var evt = new TimelineEvent(
|
||||
eventId: GenerateEventId(tenant, providerId, VexTimelineEventTypes.LinksetUpdated),
|
||||
tenant: tenant,
|
||||
providerId: providerId,
|
||||
streamId: streamId,
|
||||
eventType: VexTimelineEventTypes.LinksetUpdated,
|
||||
traceId: traceId,
|
||||
justificationSummary: justificationSummary,
|
||||
createdAt: _timeProvider.GetUtcNow(),
|
||||
evidenceHash: null,
|
||||
payloadHash: payloadHash,
|
||||
attributes: eventAttributes);
|
||||
|
||||
await EmitAsync(evt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask EmitAsync(
|
||||
TimelineEvent evt,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(evt);
|
||||
|
||||
try
|
||||
{
|
||||
var eventId = await _store.InsertAsync(evt, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Timeline event emitted: {EventType} for tenant {Tenant}, provider {ProviderId}, trace {TraceId}",
|
||||
evt.EventType,
|
||||
evt.Tenant,
|
||||
evt.ProviderId,
|
||||
evt.TraceId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to emit timeline event {EventType} for tenant {Tenant}, provider {ProviderId}: {Message}",
|
||||
evt.EventType,
|
||||
evt.Tenant,
|
||||
evt.ProviderId,
|
||||
ex.Message);
|
||||
|
||||
// Don't throw - timeline events are non-critical and shouldn't block main operations
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask EmitBatchAsync(
|
||||
string tenant,
|
||||
IEnumerable<TimelineEvent> events,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(events);
|
||||
|
||||
var eventList = events.ToList();
|
||||
if (eventList.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var insertedCount = await _store.InsertManyAsync(tenant, eventList, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Batch timeline events emitted: {InsertedCount}/{TotalCount} for tenant {Tenant}",
|
||||
insertedCount,
|
||||
eventList.Count,
|
||||
tenant);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to emit batch timeline events for tenant {Tenant}: {Message}",
|
||||
tenant,
|
||||
ex.Message);
|
||||
|
||||
// Don't throw - timeline events are non-critical
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a deterministic event ID based on tenant, provider, event type, and timestamp.
|
||||
/// </summary>
|
||||
private string GenerateEventId(string tenant, string providerId, string eventType)
|
||||
{
|
||||
var timestamp = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
var input = $"{tenant}|{providerId}|{eventType}|{timestamp}|{Guid.NewGuid():N}";
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input));
|
||||
return $"evt:{Convert.ToHexString(hash).ToLowerInvariant()[..32]}";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user