Implement Advisory Canonicalization and Backfill Migration
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added AdvisoryCanonicalizer for canonicalizing advisory identifiers.
- Created EnsureAdvisoryCanonicalKeyBackfillMigration to populate advisory_key and links in advisory_raw documents.
- Introduced FileSurfaceManifestStore for managing surface manifests with file system backing.
- Developed ISurfaceManifestReader and ISurfaceManifestWriter interfaces for reading and writing manifests.
- Implemented SurfaceManifestPathBuilder for constructing paths and URIs for surface manifests.
- Added tests for FileSurfaceManifestStore to ensure correct functionality and deterministic behavior.
- Updated documentation for new features and migration steps.
This commit is contained in:
master
2025-11-07 19:54:02 +02:00
parent a1ce3f74fa
commit 515975edc5
42 changed files with 1893 additions and 336 deletions

View File

@@ -1,10 +1,12 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
using Xunit;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
using Xunit;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Core.Tests.Linksets;
@@ -115,11 +117,11 @@ public sealed class AdvisoryObservationFactoryTests
}
[Fact]
public void Create_StoresNotesAsAttributes()
{
var factory = new AdvisoryObservationFactory();
var notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
{
public void Create_StoresNotesAsAttributes()
{
var factory = new AdvisoryObservationFactory();
var notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
{
["range-introduced"] = "1.0.0",
["range-fixed"] = "1.0.5"
});
@@ -142,7 +144,62 @@ public sealed class AdvisoryObservationFactoryTests
Assert.Equal(notes, observation.RawLinkset.Notes);
Assert.Equal(new[] { "connector-a", "connector-b" }, observation.RawLinkset.ReconciledFrom);
}
[Fact]
public void Create_IsDeterministicAcrossRuns()
{
var factory = new AdvisoryObservationFactory();
var retrievedAt = DateTimeOffset.Parse("2025-02-11T04:05:06Z");
var upstream = new RawUpstreamMetadata(
UpstreamId: "CVE-2025-1000",
DocumentVersion: "2025.02.11",
RetrievedAt: retrievedAt,
ContentHash: "sha256:deterministic-1",
Signature: new RawSignatureMetadata(true, "dsse", "key-123", "signature-data"),
Provenance: ImmutableDictionary.CreateRange(new Dictionary<string, string>
{
["api"] = "https://api.vendor.test/v1/feed",
["snapshot"] = "2025-02-11"
}));
var linkset = new RawLinkset
{
Aliases = ImmutableArray.Create("Vendor-1000", "CVE-2025-1000"),
PackageUrls = ImmutableArray.Create("pkg:npm/demo@1.0.0", "pkg:npm/demo@1.0.0"),
Cpes = ImmutableArray.Create("cpe:2.3:a:vendor:demo:1.0:*:*:*:*:*:*:*"),
References = ImmutableArray.Create(
new RawReference("advisory", "https://vendor.test/advisory", "vendor"),
new RawReference("fix", "https://vendor.test/fix", null)),
ReconciledFrom = ImmutableArray.Create("connector-y"),
Notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
{
["alias.vendor"] = "Vendor-1000"
})
};
var rawDocument = BuildRawDocument(
source: new RawSourceMetadata("vendor", "connector-y", "5.6.7", "stable"),
upstream: upstream,
identifiers: new RawIdentifiers(
Aliases: ImmutableArray.Create("CVE-2025-1000", "Vendor-1000"),
PrimaryId: "CVE-2025-1000"),
linkset: linkset,
tenant: "tenant-a");
var first = factory.Create(rawDocument, observedAt: retrievedAt);
var second = factory.Create(rawDocument, observedAt: retrievedAt);
var firstJson = CanonicalJsonSerializer.Serialize(first);
var secondJson = CanonicalJsonSerializer.Serialize(second);
Assert.Equal(firstJson, secondJson);
Assert.Equal(first.ObservationId, second.ObservationId);
Assert.True(first.Linkset.Aliases.SequenceEqual(second.Linkset.Aliases));
Assert.True(first.RawLinkset.Aliases.SequenceEqual(second.RawLinkset.Aliases));
Assert.Equal(first.CreatedAt, second.CreatedAt);
}
private static AdvisoryRawDocument BuildRawDocument(
RawSourceMetadata? source = null,
RawUpstreamMetadata? upstream = null,

View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json;
@@ -22,16 +23,19 @@ public sealed class AdvisoryRawServiceTests
var repository = new RecordingRepository();
var service = CreateService(repository);
var document = CreateDocument() with { Supersedes = " previous-id " };
var storedDocument = document.WithSupersedes("advisory_raw:vendor-x:ghsa-xxxx:sha256-2");
var expectedResult = new AdvisoryRawUpsertResult(true, CreateRecord(storedDocument));
repository.NextResult = expectedResult;
var result = await service.IngestAsync(document, CancellationToken.None);
Assert.NotNull(repository.CapturedDocument);
Assert.Null(repository.CapturedDocument!.Supersedes);
Assert.Equal(expectedResult.Record.Document.Supersedes, result.Record.Document.Supersedes);
var document = CreateDocument() with { Supersedes = " previous-id " };
var storedDocument = document.WithSupersedes("advisory_raw:vendor-x:ghsa-xxxx:sha256-2");
var expectedResult = new AdvisoryRawUpsertResult(true, CreateRecord(storedDocument));
repository.NextResult = expectedResult;
var result = await service.IngestAsync(document, CancellationToken.None);
Assert.NotNull(repository.CapturedDocument);
Assert.Null(repository.CapturedDocument!.Supersedes);
Assert.Equal(expectedResult.Record.Document.Supersedes, result.Record.Document.Supersedes);
Assert.Equal("GHSA-XXXX", repository.CapturedDocument.AdvisoryKey);
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "GHSA" && link.Value == "GHSA-XXXX");
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "PRIMARY" && link.Value == "GHSA-XXXX");
}
[Fact]
@@ -68,6 +72,31 @@ public sealed class AdvisoryRawServiceTests
Assert.NotNull(repository.CapturedDocument);
Assert.True(aliasSeries.SequenceEqual(repository.CapturedDocument!.Identifiers.Aliases));
Assert.Equal("CVE-2025-0001", repository.CapturedDocument.AdvisoryKey);
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "CVE" && link.Value == "CVE-2025-0001");
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "GHSA" && link.Value == "GHSA-XXXX");
}
[Fact]
public async Task FindByAdvisoryKeyAsync_NormalizesKeyAndVendors()
{
var repository = new RecordingRepository
{
AdvisoryKeyResults = new[] { CreateRecord(CreateDocument()) }
};
var service = CreateService(repository);
var results = await service.FindByAdvisoryKeyAsync(
"Tenant-Example",
"ghsa-xxxx",
new[] { "Vendor-X", " " },
CancellationToken.None);
Assert.Single(results);
Assert.Equal("tenant-example", repository.CapturedTenant);
Assert.Contains("GHSA-XXXX", repository.CapturedAdvisoryKeySearchValues!, StringComparer.Ordinal);
Assert.Contains("ghsa-xxxx", repository.CapturedAdvisoryKeySearchValues!, StringComparer.Ordinal);
Assert.Contains("vendor-x", repository.CapturedAdvisoryKeyVendors!, StringComparer.Ordinal);
}
private static AdvisoryRawService CreateService(RecordingRepository repository)
@@ -86,56 +115,75 @@ public sealed class AdvisoryRawServiceTests
private static AdvisoryRawDocument CreateDocument()
{
using var raw = JsonDocument.Parse("""{"id":"demo"}""");
return new AdvisoryRawDocument(
Tenant: "Tenant-A",
Source: new RawSourceMetadata("Vendor-X", "connector-y", "1.0.0"),
Upstream: new RawUpstreamMetadata(
UpstreamId: "GHSA-xxxx",
DocumentVersion: "1",
RetrievedAt: DateTimeOffset.UtcNow,
ContentHash: "sha256:abc",
Signature: new RawSignatureMetadata(
Present: true,
Format: "dsse",
KeyId: "key-1",
Signature: "base64signature"),
Provenance: ImmutableDictionary<string, string>.Empty),
Content: new RawContent(
Format: "OSV",
SpecVersion: "1.0",
Raw: raw.RootElement.Clone()),
Identifiers: new RawIdentifiers(
Aliases: ImmutableArray.Create("GHSA-xxxx"),
PrimaryId: "GHSA-xxxx"),
Linkset: new RawLinkset
{
Aliases = ImmutableArray<string>.Empty,
PackageUrls = ImmutableArray<string>.Empty,
Cpes = ImmutableArray<string>.Empty,
References = ImmutableArray<RawReference>.Empty,
ReconciledFrom = ImmutableArray<string>.Empty,
Notes = ImmutableDictionary<string, string>.Empty
});
}
return new AdvisoryRawDocument(
Tenant: "Tenant-A",
Source: new RawSourceMetadata("Vendor-X", "connector-y", "1.0.0"),
Upstream: new RawUpstreamMetadata(
UpstreamId: "GHSA-xxxx",
DocumentVersion: "1",
RetrievedAt: DateTimeOffset.UtcNow,
ContentHash: "sha256:abc",
Signature: new RawSignatureMetadata(
Present: true,
Format: "dsse",
KeyId: "key-1",
Signature: "base64signature"),
Provenance: ImmutableDictionary<string, string>.Empty),
Content: new RawContent(
Format: "OSV",
SpecVersion: "1.0",
Raw: raw.RootElement.Clone()),
Identifiers: new RawIdentifiers(
Aliases: ImmutableArray.Create("GHSA-xxxx"),
PrimaryId: "GHSA-xxxx"),
Linkset: new RawLinkset
{
Aliases = ImmutableArray<string>.Empty,
PackageUrls = ImmutableArray<string>.Empty,
Cpes = ImmutableArray<string>.Empty,
References = ImmutableArray<RawReference>.Empty,
ReconciledFrom = ImmutableArray<string>.Empty,
Notes = ImmutableDictionary<string, string>.Empty
},
AdvisoryKey: string.Empty,
Links: ImmutableArray<RawLink>.Empty);
}
private static AdvisoryRawRecord CreateRecord(AdvisoryRawDocument document)
=> new(
Id: "advisory_raw:vendor-x:ghsa-xxxx:sha256-1",
Document: document,
IngestedAt: DateTimeOffset.UtcNow,
CreatedAt: document.Upstream.RetrievedAt);
private static AdvisoryRawRecord CreateRecord(AdvisoryRawDocument document)
{
var canonical = AdvisoryCanonicalizer.Canonicalize(document.Identifiers, document.Source, document.Upstream);
var resolvedDocument = document with
{
AdvisoryKey = string.IsNullOrWhiteSpace(document.AdvisoryKey) ? canonical.AdvisoryKey : document.AdvisoryKey,
Links = document.Links.IsDefaultOrEmpty ? canonical.Links : document.Links
};
return new AdvisoryRawRecord(
Id: "advisory_raw:vendor-x:ghsa-xxxx:sha256-1",
Document: resolvedDocument,
IngestedAt: DateTimeOffset.UtcNow,
CreatedAt: document.Upstream.RetrievedAt);
}
private sealed class RecordingRepository : IAdvisoryRawRepository
{
public AdvisoryRawDocument? CapturedDocument { get; private set; }
public AdvisoryRawUpsertResult? NextResult { get; set; }
public Task<AdvisoryRawUpsertResult> UpsertAsync(AdvisoryRawDocument document, CancellationToken cancellationToken)
{
if (NextResult is null)
{
throw new InvalidOperationException("NextResult must be set before calling UpsertAsync.");
{
public AdvisoryRawDocument? CapturedDocument { get; private set; }
public AdvisoryRawUpsertResult? NextResult { get; set; }
public string? CapturedTenant { get; private set; }
public IReadOnlyCollection<string>? CapturedAdvisoryKeySearchValues { get; private set; }
public IReadOnlyCollection<string>? CapturedAdvisoryKeyVendors { get; private set; }
public IReadOnlyList<AdvisoryRawRecord> AdvisoryKeyResults { get; set; } = Array.Empty<AdvisoryRawRecord>();
public Task<AdvisoryRawUpsertResult> UpsertAsync(AdvisoryRawDocument document, CancellationToken cancellationToken)
{
if (NextResult is null)
{
throw new InvalidOperationException("NextResult must be set before calling UpsertAsync.");
}
CapturedDocument = document;
@@ -145,14 +193,26 @@ public sealed class AdvisoryRawServiceTests
public Task<AdvisoryRawRecord?> FindByIdAsync(string tenant, string id, CancellationToken cancellationToken)
=> throw new NotSupportedException();
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
=> throw new NotSupportedException();
public Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
string tenant,
DateTimeOffset since,
DateTimeOffset until,
IReadOnlyCollection<string> sourceVendors,
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
=> throw new NotSupportedException();
public Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
string tenant,
IReadOnlyCollection<string> searchValues,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken)
{
CapturedTenant = tenant;
CapturedAdvisoryKeySearchValues = searchValues?.ToArray();
CapturedAdvisoryKeyVendors = sourceVendors?.ToArray();
return Task.FromResult(AdvisoryKeyResults);
}
public Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
string tenant,
DateTimeOffset since,
DateTimeOffset until,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken)
=> throw new NotSupportedException();
}

View File

@@ -1,5 +1,4 @@
using System;
using System.Linq;
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
@@ -76,29 +75,6 @@ public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime
Assert.True(persisted.BeforeHash.Length > 0);
}
[Fact]
public async Task MergePipeline_IsDeterministicAcrossRuns()
{
await EnsureInitializedAsync();
var merger = _merger!;
var calculator = new CanonicalHashCalculator();
var firstResult = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() });
var secondResult = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() });
var first = firstResult.Advisory;
var second = secondResult.Advisory;
var firstHash = calculator.ComputeHash(first);
var secondHash = calculator.ComputeHash(second);
Assert.Equal(firstHash, secondHash);
Assert.Equal(first.AdvisoryKey, second.AdvisoryKey);
Assert.Equal(first.Aliases.Length, second.Aliases.Length);
Assert.True(first.Aliases.SequenceEqual(second.Aliases));
}
public async Task InitializeAsync()
{
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero))

View File

@@ -69,7 +69,12 @@ public sealed class EnsureAdvisoryObservationsRawLinksetMigrationTests
References = ImmutableArray.Create(new RawReference("advisory", "https://example.test/advisory", "vendor")),
ReconciledFrom = ImmutableArray.Create("connector-y"),
Notes = ImmutableDictionary.CreateRange(new[] { new KeyValuePair<string, string>("range-fixed", "1.0.1") })
});
},
advisoryKey: "CVE-2025-0001",
links: ImmutableArray.Create(
new RawLink("CVE", "CVE-2025-0001"),
new RawLink("GHSA", "GHSA-2025-0001"),
new RawLink("PRIMARY", "CVE-2025-0001")));
await rawRepository.UpsertAsync(rawDocument, CancellationToken.None);
@@ -147,7 +152,11 @@ public sealed class EnsureAdvisoryObservationsRawLinksetMigrationTests
identifiers: new RawIdentifiers(
Aliases: ImmutableArray<string>.Empty,
PrimaryId: "GHSA-9999-0001"),
linkset: new RawLinkset());
linkset: new RawLinkset(),
advisoryKey: "GHSA-9999-0001",
links: ImmutableArray.Create(
new RawLink("GHSA", "GHSA-9999-0001"),
new RawLink("PRIMARY", "GHSA-9999-0001")));
var observationId = "tenant-b:vendor-y:ghsa-9999-0001:sha256-def456";
var document = BuildObservationDocument(

View File

@@ -686,6 +686,21 @@ public sealed class MongoMigrationRunnerTests
{ "notes", new BsonDocument() },
}
},
{ "advisory_key", upstreamId.ToUpperInvariant() },
{
"links",
new BsonArray
{
new BsonDocument
{
{ "scheme", "PRIMARY" },
{ "value", upstreamId.ToUpperInvariant() }
}
}
},
{ "created_at", retrievedAt },
{ "ingested_at", retrievedAt },
{ "supersedes", BsonNull.Value }
};
}
}

View File

@@ -8,11 +8,11 @@ namespace StellaOps.Concelier.WebService.Tests;
public sealed class ConcelierOptionsPostConfigureTests
{
[Fact]
public void Apply_LoadsClientSecretFromRelativeFile()
{
var tempDirectory = Directory.CreateTempSubdirectory();
try
{
public void Apply_LoadsClientSecretFromRelativeFile()
{
var tempDirectory = Directory.CreateTempSubdirectory();
try
{
var secretPath = Path.Combine(tempDirectory.FullName, "authority.secret");
File.WriteAllText(secretPath, " concelier-secret ");
@@ -34,14 +34,22 @@ public sealed class ConcelierOptionsPostConfigureTests
{
Directory.Delete(tempDirectory.FullName, recursive: true);
}
}
}
[Fact]
public void Apply_ThrowsWhenSecretFileMissing()
{
var options = new ConcelierOptions
{
}
}
[Fact]
public void Features_NoMergeEnabled_DefaultsToTrue()
{
var options = new ConcelierOptions();
Assert.True(options.Features.NoMergeEnabled);
}
[Fact]
public void Apply_ThrowsWhenSecretFileMissing()
{
var options = new ConcelierOptions
{
Authority = new ConcelierOptions.AuthorityOptions
{
ClientSecretFile = "missing.secret"

View File

@@ -469,6 +469,55 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Assert.Empty(firstIds.Intersect(secondIds));
}
[Fact]
public async Task AdvisoryEvidenceEndpoint_ReturnsDocumentsForCanonicalKey()
{
await SeedAdvisoryRawDocumentsAsync(
CreateAdvisoryRawDocument("tenant-a", "vendor-x", "GHSA-2025-0001", "sha256:001", new BsonDocument("id", "GHSA-2025-0001:1")),
CreateAdvisoryRawDocument("tenant-a", "vendor-y", "GHSA-2025-0001", "sha256:002", new BsonDocument("id", "GHSA-2025-0001:2")),
CreateAdvisoryRawDocument("tenant-b", "vendor-x", "GHSA-2025-0001", "sha256:003", new BsonDocument("id", "GHSA-2025-0001:3")));
using var client = _factory.CreateClient();
var response = await client.GetAsync("/vuln/evidence/advisories/ghsa-2025-0001?tenant=tenant-a");
response.EnsureSuccessStatusCode();
var evidence = await response.Content.ReadFromJsonAsync<AdvisoryEvidenceResponse>();
Assert.NotNull(evidence);
Assert.Equal("GHSA-2025-0001", evidence!.AdvisoryKey);
Assert.Equal(2, evidence.Records.Count);
Assert.All(evidence.Records, record => Assert.Equal("tenant-a", record.Tenant));
}
[Fact]
public async Task AdvisoryEvidenceEndpoint_FiltersByVendor()
{
await SeedAdvisoryRawDocumentsAsync(
CreateAdvisoryRawDocument("tenant-a", "vendor-x", "GHSA-2025-0002", "sha256:101", new BsonDocument("id", "GHSA-2025-0002:1")),
CreateAdvisoryRawDocument("tenant-a", "vendor-y", "GHSA-2025-0002", "sha256:102", new BsonDocument("id", "GHSA-2025-0002:2")));
using var client = _factory.CreateClient();
var response = await client.GetAsync("/vuln/evidence/advisories/GHSA-2025-0002?tenant=tenant-a&vendor=vendor-y");
response.EnsureSuccessStatusCode();
var evidence = await response.Content.ReadFromJsonAsync<AdvisoryEvidenceResponse>();
Assert.NotNull(evidence);
var record = Assert.Single(evidence!.Records);
Assert.Equal("vendor-y", record.Document.Source.Vendor);
}
[Fact]
public async Task AdvisoryEvidenceEndpoint_ReturnsNotFoundWhenMissing()
{
await SeedAdvisoryRawDocumentsAsync();
using var client = _factory.CreateClient();
var response = await client.GetAsync("/vuln/evidence/advisories/CVE-2099-9999?tenant=tenant-a");
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
}
[Fact]
public async Task AdvisoryIngestEndpoint_EmitsMetricsWithExpectedTags()
{
@@ -1871,6 +1920,18 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
{ "notes", new BsonDocument() }
}
},
{ "advisory_key", upstreamId.ToUpperInvariant() },
{
"links",
new BsonArray
{
new BsonDocument
{
{ "scheme", "PRIMARY" },
{ "value", upstreamId.ToUpperInvariant() }
}
}
},
{ "supersedes", supersedes is null ? BsonNull.Value : supersedes },
{ "ingested_at", now },
{ "created_at", now }