Rename Vexer to Excititor

This commit is contained in:
2025-10-18 20:00:46 +03:00
parent fbd1826ef3
commit 7e1b10d3b2
263 changed files with 848 additions and 848 deletions

View File

@@ -0,0 +1,122 @@
using System.Collections.Generic;
using Microsoft.Extensions.Logging.Abstractions;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly IMongoDatabase _database;
public MongoVexCacheMaintenanceTests()
{
_runner = MongoDbRunner.Start();
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase("vex-cache-maintenance-tests");
VexMongoMappingRegistry.Register();
}
[Fact]
public async Task RemoveExpiredAsync_DeletesEntriesBeforeCutoff()
{
var collection = _database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache);
var now = DateTime.UtcNow;
await collection.InsertManyAsync(new[]
{
new VexCacheEntryRecord
{
Id = "sig-1|json",
QuerySignature = "sig-1",
Format = "json",
ArtifactAlgorithm = "sha256",
ArtifactDigest = "deadbeef",
CreatedAt = now.AddHours(-2),
ExpiresAt = now.AddHours(-1),
},
new VexCacheEntryRecord
{
Id = "sig-2|json",
QuerySignature = "sig-2",
Format = "json",
ArtifactAlgorithm = "sha256",
ArtifactDigest = "cafebabe",
CreatedAt = now,
ExpiresAt = now.AddHours(1),
},
});
var maintenance = new MongoVexCacheMaintenance(_database, NullLogger<MongoVexCacheMaintenance>.Instance);
var removed = await maintenance.RemoveExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None);
Assert.Equal(1, removed);
var remaining = await collection.CountDocumentsAsync(FilterDefinition<VexCacheEntryRecord>.Empty);
Assert.Equal(1, remaining);
}
[Fact]
public async Task RemoveMissingManifestReferencesAsync_DropsDanglingEntries()
{
var cache = _database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache);
var exports = _database.GetCollection<VexExportManifestRecord>(VexMongoCollectionNames.Exports);
await exports.InsertOneAsync(new VexExportManifestRecord
{
Id = "manifest-existing",
QuerySignature = "sig-keep",
Format = "json",
CreatedAt = DateTime.UtcNow,
ArtifactAlgorithm = "sha256",
ArtifactDigest = "keep",
ClaimCount = 1,
SourceProviders = new List<string> { "vendor" },
});
await cache.InsertManyAsync(new[]
{
new VexCacheEntryRecord
{
Id = "sig-remove|json",
QuerySignature = "sig-remove",
Format = "json",
ArtifactAlgorithm = "sha256",
ArtifactDigest = "drop",
CreatedAt = DateTime.UtcNow,
ManifestId = "manifest-missing",
},
new VexCacheEntryRecord
{
Id = "sig-keep|json",
QuerySignature = "sig-keep",
Format = "json",
ArtifactAlgorithm = "sha256",
ArtifactDigest = "keep",
CreatedAt = DateTime.UtcNow,
ManifestId = "manifest-existing",
},
});
var maintenance = new MongoVexCacheMaintenance(_database, NullLogger<MongoVexCacheMaintenance>.Instance);
var removed = await maintenance.RemoveMissingManifestReferencesAsync(CancellationToken.None);
Assert.Equal(1, removed);
var remainingIds = await cache.Find(Builders<VexCacheEntryRecord>.Filter.Empty)
.Project(x => x.Id)
.ToListAsync();
Assert.Single(remainingIds);
Assert.Contains("sig-keep|json", remainingIds);
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
}

View File

@@ -0,0 +1,206 @@
using System.Collections.Immutable;
using System.Globalization;
using System.Text;
using Microsoft.Extensions.Options;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexRepositoryTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly MongoClient _client;
public MongoVexRepositoryTests()
{
_runner = MongoDbRunner.Start();
_client = new MongoClient(_runner.ConnectionString);
}
[Fact]
public async Task RawStore_UsesGridFsForLargePayloads()
{
var database = _client.GetDatabase($"vex-raw-gridfs-{Guid.NewGuid():N}");
var store = CreateRawStore(database, thresholdBytes: 32);
var payload = Encoding.UTF8.GetBytes(new string('A', 256));
var document = new VexRawDocument(
"red-hat",
VexDocumentFormat.Csaf,
new Uri("https://example.com/redhat/csaf.json"),
DateTimeOffset.UtcNow,
"sha256:large",
payload,
ImmutableDictionary<string, string>.Empty);
await store.StoreAsync(document, CancellationToken.None);
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var stored = await rawCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", document.Digest))
.FirstOrDefaultAsync();
Assert.NotNull(stored);
Assert.True(stored!.TryGetValue("GridFsObjectId", out var gridId));
Assert.False(gridId.IsBsonNull);
Assert.Empty(stored["Content"].AsBsonBinaryData.Bytes);
var filesCollection = database.GetCollection<BsonDocument>("vex.raw.files");
var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
Assert.Equal(1, fileCount);
var fetched = await store.FindByDigestAsync(document.Digest, CancellationToken.None);
Assert.NotNull(fetched);
Assert.Equal(payload, fetched!.Content.ToArray());
}
[Fact]
public async Task RawStore_ReplacesGridFsWithInlinePayload()
{
var database = _client.GetDatabase($"vex-raw-inline-{Guid.NewGuid():N}");
var store = CreateRawStore(database, thresholdBytes: 16);
var largePayload = Encoding.UTF8.GetBytes(new string('B', 128));
var digest = "sha256:inline";
var largeDocument = new VexRawDocument(
"cisco",
VexDocumentFormat.CycloneDx,
new Uri("https://example.com/cyclonedx.json"),
DateTimeOffset.UtcNow,
digest,
largePayload,
ImmutableDictionary<string, string>.Empty);
await store.StoreAsync(largeDocument, CancellationToken.None);
var smallDocument = largeDocument with
{
RetrievedAt = DateTimeOffset.UtcNow.AddMinutes(1),
Content = Encoding.UTF8.GetBytes("small"),
};
await store.StoreAsync(smallDocument, CancellationToken.None);
var rawCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var stored = await rawCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", digest))
.FirstOrDefaultAsync();
Assert.NotNull(stored);
Assert.True(stored!.TryGetValue("GridFsObjectId", out var gridId));
Assert.True(gridId.IsBsonNull);
Assert.Equal("small", Encoding.UTF8.GetString(stored["Content"].AsBsonBinaryData.Bytes));
var filesCollection = database.GetCollection<BsonDocument>("vex.raw.files");
var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
Assert.Equal(0, fileCount);
}
[Fact]
public async Task ExportStore_SavesManifestAndCacheTransactionally()
{
var database = _client.GetDatabase($"vex-export-save-{Guid.NewGuid():N}");
var options = Options.Create(new VexMongoStorageOptions
{
ExportCacheTtl = TimeSpan.FromHours(6),
GridFsInlineThresholdBytes = 64,
});
var store = new MongoVexExportStore(_client, database, options);
var signature = new VexQuerySignature("format=csaf|provider=redhat");
var manifest = new VexExportManifest(
"exports/20251016/redhat",
signature,
VexExportFormat.Csaf,
DateTimeOffset.UtcNow,
new VexContentAddress("sha256", "abcdef123456"),
claimCount: 5,
sourceProviders: new[] { "red-hat" },
fromCache: false,
consensusRevision: "rev-1",
attestation: null,
sizeBytes: 1024);
await store.SaveAsync(manifest, CancellationToken.None);
var exportsCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Exports);
var exportKey = BuildExportKey(signature, VexExportFormat.Csaf);
var exportDoc = await exportsCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", exportKey))
.FirstOrDefaultAsync();
Assert.NotNull(exportDoc);
var cacheCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Cache);
var cacheKey = BuildExportKey(signature, VexExportFormat.Csaf);
var cacheDoc = await cacheCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", cacheKey))
.FirstOrDefaultAsync();
Assert.NotNull(cacheDoc);
Assert.Equal(manifest.ExportId, cacheDoc!["ManifestId"].AsString);
Assert.True(cacheDoc.TryGetValue("ExpiresAt", out var expiresValue));
Assert.False(expiresValue.IsBsonNull);
}
[Fact]
public async Task ExportStore_FindAsync_ExpiresCacheEntries()
{
var database = _client.GetDatabase($"vex-export-expire-{Guid.NewGuid():N}");
var options = Options.Create(new VexMongoStorageOptions
{
ExportCacheTtl = TimeSpan.FromMinutes(5),
GridFsInlineThresholdBytes = 64,
});
var store = new MongoVexExportStore(_client, database, options);
var signature = new VexQuerySignature("format=json|provider=cisco");
var manifest = new VexExportManifest(
"exports/20251016/cisco",
signature,
VexExportFormat.Json,
DateTimeOffset.UtcNow,
new VexContentAddress("sha256", "deadbeef"),
claimCount: 3,
sourceProviders: new[] { "cisco" },
fromCache: false,
consensusRevision: "rev-2",
attestation: null,
sizeBytes: 2048);
await store.SaveAsync(manifest, CancellationToken.None);
var cacheCollection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Cache);
var cacheId = BuildExportKey(signature, VexExportFormat.Json);
var update = Builders<BsonDocument>.Update.Set("ExpiresAt", DateTime.UtcNow.AddMinutes(-10));
await cacheCollection.UpdateOneAsync(Builders<BsonDocument>.Filter.Eq("_id", cacheId), update);
var cached = await store.FindAsync(signature, VexExportFormat.Json, CancellationToken.None);
Assert.Null(cached);
var remaining = await cacheCollection.Find(Builders<BsonDocument>.Filter.Eq("_id", cacheId))
.FirstOrDefaultAsync();
Assert.Null(remaining);
}
private MongoVexRawStore CreateRawStore(IMongoDatabase database, int thresholdBytes)
{
var options = Options.Create(new VexMongoStorageOptions
{
RawBucketName = "vex.raw",
GridFsInlineThresholdBytes = thresholdBytes,
ExportCacheTtl = TimeSpan.FromHours(1),
});
return new MongoVexRawStore(_client, database, options);
}
private static string BuildExportKey(VexQuerySignature signature, VexExportFormat format)
=> string.Format(CultureInfo.InvariantCulture, "{0}|{1}", signature.Value, format.ToString().ToLowerInvariant());
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
}

View File

@@ -0,0 +1,242 @@
using System.Globalization;
using Mongo2Go;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class MongoVexStoreMappingTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly IMongoDatabase _database;
public MongoVexStoreMappingTests()
{
_runner = MongoDbRunner.Start();
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase("excititor-storage-mapping-tests");
VexMongoMappingRegistry.Register();
}
[Fact]
public async Task ProviderStore_RoundTrips_WithExtraFields()
{
var providers = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Providers);
var providerId = "red-hat";
var document = new BsonDocument
{
{ "_id", providerId },
{ "DisplayName", "Red Hat CSAF" },
{ "Kind", "vendor" },
{ "BaseUris", new BsonArray { "https://example.com/csaf" } },
{
"Discovery",
new BsonDocument
{
{ "WellKnownMetadata", "https://example.com/.well-known/csaf" },
{ "RolIeService", "https://example.com/service/rolie" },
{ "UnsupportedField", "ignored" },
}
},
{
"Trust",
new BsonDocument
{
{ "Weight", 0.75 },
{
"Cosign",
new BsonDocument
{
{ "Issuer", "issuer@example.com" },
{ "IdentityPattern", "spiffe://example/*" },
{ "Unexpected", true },
}
},
{ "PgpFingerprints", new BsonArray { "ABCDEF1234567890" } },
{ "AnotherIgnoredField", 123 },
}
},
{ "Enabled", true },
{ "UnexpectedRoot", new BsonDocument { { "flag", true } } },
};
await providers.InsertOneAsync(document);
var store = new MongoVexProviderStore(_database);
var result = await store.FindAsync(providerId, CancellationToken.None);
Assert.NotNull(result);
Assert.Equal(providerId, result!.Id);
Assert.Equal("Red Hat CSAF", result.DisplayName);
Assert.Equal(VexProviderKind.Vendor, result.Kind);
Assert.Single(result.BaseUris);
Assert.Equal("https://example.com/csaf", result.BaseUris[0].ToString());
Assert.Equal("https://example.com/.well-known/csaf", result.Discovery.WellKnownMetadata?.ToString());
Assert.Equal("https://example.com/service/rolie", result.Discovery.RolIeService?.ToString());
Assert.Equal(0.75, result.Trust.Weight);
Assert.NotNull(result.Trust.Cosign);
Assert.Equal("issuer@example.com", result.Trust.Cosign!.Issuer);
Assert.Equal("spiffe://example/*", result.Trust.Cosign!.IdentityPattern);
Assert.Contains("ABCDEF1234567890", result.Trust.PgpFingerprints);
Assert.True(result.Enabled);
}
[Fact]
public async Task ConsensusStore_IgnoresUnknownFields()
{
var consensus = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
var vulnerabilityId = "CVE-2025-12345";
var productKey = "pkg:maven/org.example/app@1.2.3";
var consensusId = string.Format(CultureInfo.InvariantCulture, "{0}|{1}", vulnerabilityId.Trim(), productKey.Trim());
var document = new BsonDocument
{
{ "_id", consensusId },
{ "VulnerabilityId", vulnerabilityId },
{
"Product",
new BsonDocument
{
{ "Key", productKey },
{ "Name", "Example App" },
{ "Version", "1.2.3" },
{ "Purl", productKey },
{ "Extra", "ignored" },
}
},
{ "Status", "notaffected" },
{ "CalculatedAt", DateTime.UtcNow },
{
"Sources",
new BsonArray
{
new BsonDocument
{
{ "ProviderId", "red-hat" },
{ "Status", "notaffected" },
{ "DocumentDigest", "sha256:123" },
{ "Weight", 0.9 },
{ "Justification", "componentnotpresent" },
{ "Detail", "Vendor statement" },
{
"Confidence",
new BsonDocument
{
{ "Level", "high" },
{ "Score", 0.7 },
{ "Method", "review" },
{ "Unexpected", "ignored" },
}
},
{ "UnknownField", true },
},
}
},
{
"Conflicts",
new BsonArray
{
new BsonDocument
{
{ "ProviderId", "cisco" },
{ "Status", "affected" },
{ "DocumentDigest", "sha256:999" },
{ "Justification", "requiresconfiguration" },
{ "Detail", "Different guidance" },
{ "Reason", "policy_override" },
{ "Other", 1 },
},
}
},
{ "PolicyVersion", "2025.10" },
{ "PolicyRevisionId", "rev-1" },
{ "PolicyDigest", "sha256:abc" },
{ "Summary", "Vendor confirms not affected." },
{ "Unexpected", new BsonDocument { { "foo", "bar" } } },
};
await consensus.InsertOneAsync(document);
var store = new MongoVexConsensusStore(_database);
var result = await store.FindAsync(vulnerabilityId, productKey, CancellationToken.None);
Assert.NotNull(result);
Assert.Equal(vulnerabilityId, result!.VulnerabilityId);
Assert.Equal(productKey, result.Product.Key);
Assert.Equal("Example App", result.Product.Name);
Assert.Equal(VexConsensusStatus.NotAffected, result.Status);
Assert.Single(result.Sources);
var source = result.Sources[0];
Assert.Equal("red-hat", source.ProviderId);
Assert.Equal(VexClaimStatus.NotAffected, source.Status);
Assert.Equal("sha256:123", source.DocumentDigest);
Assert.Equal(0.9, source.Weight);
Assert.Equal(VexJustification.ComponentNotPresent, source.Justification);
Assert.NotNull(source.Confidence);
Assert.Equal("high", source.Confidence!.Level);
Assert.Equal(0.7, source.Confidence!.Score);
Assert.Equal("review", source.Confidence!.Method);
Assert.Single(result.Conflicts);
var conflict = result.Conflicts[0];
Assert.Equal("cisco", conflict.ProviderId);
Assert.Equal(VexClaimStatus.Affected, conflict.Status);
Assert.Equal(VexJustification.RequiresConfiguration, conflict.Justification);
Assert.Equal("policy_override", conflict.Reason);
Assert.Equal("Vendor confirms not affected.", result.Summary);
Assert.Equal("2025.10", result.PolicyVersion);
}
[Fact]
public async Task CacheIndex_RoundTripsGridFsMetadata()
{
var gridObjectId = ObjectId.GenerateNewId().ToString();
var index = new MongoVexCacheIndex(_database);
var signature = new VexQuerySignature("format=csaf|vendor=redhat");
var now = DateTimeOffset.UtcNow;
var expires = now.AddHours(12);
var entry = new VexCacheEntry(
signature,
VexExportFormat.Csaf,
new VexContentAddress("sha256", "abcdef123456"),
now,
sizeBytes: 1024,
manifestId: "manifest-001",
gridFsObjectId: gridObjectId,
expiresAt: expires);
await index.SaveAsync(entry, CancellationToken.None);
var cacheId = string.Format(
CultureInfo.InvariantCulture,
"{0}|{1}",
signature.Value,
entry.Format.ToString().ToLowerInvariant());
var cache = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Cache);
var filter = Builders<BsonDocument>.Filter.Eq("_id", cacheId);
var update = Builders<BsonDocument>.Update.Set("UnexpectedField", true);
await cache.UpdateOneAsync(filter, update);
var roundTrip = await index.FindAsync(signature, VexExportFormat.Csaf, CancellationToken.None);
Assert.NotNull(roundTrip);
Assert.Equal(entry.QuerySignature.Value, roundTrip!.QuerySignature.Value);
Assert.Equal(entry.Format, roundTrip.Format);
Assert.Equal(entry.Artifact.Digest, roundTrip.Artifact.Digest);
Assert.Equal(entry.ManifestId, roundTrip.ManifestId);
Assert.Equal(entry.GridFsObjectId, roundTrip.GridFsObjectId);
Assert.Equal(entry.SizeBytes, roundTrip.SizeBytes);
Assert.NotNull(roundTrip.ExpiresAt);
Assert.Equal(expires.ToUnixTimeMilliseconds(), roundTrip.ExpiresAt!.Value.ToUnixTimeMilliseconds());
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
}

View File

@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Policy\StellaOps.Excititor.Policy.csproj" />
<ProjectReference Include="..\StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,59 @@
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Mongo2Go;
using MongoDB.Driver;
using StellaOps.Excititor.Storage.Mongo.Migrations;
namespace StellaOps.Excititor.Storage.Mongo.Tests;
public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime
{
private readonly MongoDbRunner _runner;
private readonly IMongoDatabase _database;
public VexMongoMigrationRunnerTests()
{
_runner = MongoDbRunner.Start();
var client = new MongoClient(_runner.ConnectionString);
_database = client.GetDatabase("excititor-migrations-tests");
}
[Fact]
public async Task RunAsync_AppliesInitialIndexesOnce()
{
var migration = new VexInitialIndexMigration();
var runner = new VexMongoMigrationRunner(_database, new[] { migration }, NullLogger<VexMongoMigrationRunner>.Instance);
await runner.RunAsync(CancellationToken.None);
await runner.RunAsync(CancellationToken.None);
var appliedCollection = _database.GetCollection<VexMigrationRecord>(VexMongoCollectionNames.Migrations);
var applied = await appliedCollection.Find(FilterDefinition<VexMigrationRecord>.Empty).ToListAsync();
Assert.Single(applied);
Assert.Equal(migration.Id, applied[0].Id);
Assert.True(HasIndex(_database.GetCollection<VexRawDocumentRecord>(VexMongoCollectionNames.Raw), "ProviderId_1_Format_1_RetrievedAt_1"));
Assert.True(HasIndex(_database.GetCollection<VexProviderRecord>(VexMongoCollectionNames.Providers), "Kind_1"));
Assert.True(HasIndex(_database.GetCollection<VexConsensusRecord>(VexMongoCollectionNames.Consensus), "VulnerabilityId_1_Product.Key_1"));
Assert.True(HasIndex(_database.GetCollection<VexConsensusRecord>(VexMongoCollectionNames.Consensus), "PolicyRevisionId_1_PolicyDigest_1"));
Assert.True(HasIndex(_database.GetCollection<VexExportManifestRecord>(VexMongoCollectionNames.Exports), "QuerySignature_1_Format_1"));
Assert.True(HasIndex(_database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache), "QuerySignature_1_Format_1"));
Assert.True(HasIndex(_database.GetCollection<VexCacheEntryRecord>(VexMongoCollectionNames.Cache), "ExpiresAt_1"));
}
private static bool HasIndex<TDocument>(IMongoCollection<TDocument> collection, string name)
{
var indexes = collection.Indexes.List().ToList();
return indexes.Any(index => index["name"].AsString == name);
}
public Task InitializeAsync() => Task.CompletedTask;
public Task DisposeAsync()
{
_runner.Dispose();
return Task.CompletedTask;
}
}