Implement Advisory Canonicalization and Backfill Migration
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added AdvisoryCanonicalizer for canonicalizing advisory identifiers.
- Created EnsureAdvisoryCanonicalKeyBackfillMigration to populate advisory_key and links in advisory_raw documents.
- Introduced FileSurfaceManifestStore for managing surface manifests with file system backing.
- Developed ISurfaceManifestReader and ISurfaceManifestWriter interfaces for reading and writing manifests.
- Implemented SurfaceManifestPathBuilder for constructing paths and URIs for surface manifests.
- Added tests for FileSurfaceManifestStore to ensure correct functionality and deterministic behavior.
- Updated documentation for new features and migration steps.
This commit is contained in:
master
2025-11-07 19:54:02 +02:00
parent a1ce3f74fa
commit 515975edc5
42 changed files with 1893 additions and 336 deletions

View File

@@ -74,16 +74,20 @@ public sealed record AdvisoryRawRecordResponse(
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
public sealed record AdvisoryRawListResponse(
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
[property: JsonPropertyName("nextCursor")] string? NextCursor,
[property: JsonPropertyName("hasMore")] bool HasMore);
public sealed record AdvisoryRawProvenanceResponse(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("source")] RawSourceMetadata Source,
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
public sealed record AdvisoryRawListResponse(
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
[property: JsonPropertyName("nextCursor")] string? NextCursor,
[property: JsonPropertyName("hasMore")] bool HasMore);
public sealed record AdvisoryEvidenceResponse(
[property: JsonPropertyName("advisoryKey")] string AdvisoryKey,
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records);
public sealed record AdvisoryRawProvenanceResponse(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("source")] RawSourceMetadata Source,
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
[property: JsonPropertyName("supersedes")] string? Supersedes,
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);

View File

@@ -61,24 +61,26 @@ internal static class AdvisoryRawRequestMapper
identifiersRequest.Primary);
var linksetRequest = request.Linkset;
var linkset = new RawLinkset
{
Aliases = NormalizeStrings(linksetRequest?.Aliases),
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
Cpes = NormalizeStrings(linksetRequest?.Cpes),
References = NormalizeReferences(linksetRequest?.References),
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
Notes = NormalizeDictionary(linksetRequest?.Notes)
};
return new AdvisoryRawDocument(
tenant.Trim().ToLowerInvariant(),
source,
upstream,
content,
identifiers,
linkset);
}
var linkset = new RawLinkset
{
Aliases = NormalizeStrings(linksetRequest?.Aliases),
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
Cpes = NormalizeStrings(linksetRequest?.Cpes),
References = NormalizeReferences(linksetRequest?.References),
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
Notes = NormalizeDictionary(linksetRequest?.Notes)
};
return new AdvisoryRawDocument(
tenant.Trim().ToLowerInvariant(),
source,
upstream,
content,
identifiers,
linkset,
AdvisoryKey: string.Empty,
Links: ImmutableArray<RawLink>.Empty);
}
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
{

View File

@@ -144,7 +144,7 @@ public sealed class ConcelierOptions
public sealed class FeaturesOptions
{
public bool NoMergeEnabled { get; set; }
public bool NoMergeEnabled { get; set; } = true;
public bool LnmShadowWrites { get; set; } = true;

View File

@@ -672,6 +672,59 @@ if (authorityConfigured)
advisoryRawProvenanceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKey}", async (
string advisoryKey,
HttpContext context,
[FromServices] IAdvisoryRawService rawService,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var authorizationError = EnsureTenantAuthorized(context, tenant);
if (authorizationError is not null)
{
return authorizationError;
}
if (string.IsNullOrWhiteSpace(advisoryKey))
{
return Problem(context, "advisoryKey is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide an advisory identifier.");
}
var vendorFilter = AdvisoryRawRequestMapper.NormalizeStrings(context.Request.Query["vendor"]);
var records = await rawService.FindByAdvisoryKeyAsync(
tenant,
advisoryKey,
vendorFilter,
cancellationToken).ConfigureAwait(false);
if (records.Count == 0)
{
return Results.NotFound();
}
var recordResponses = records
.Select(record => new AdvisoryRawRecordResponse(
record.Id,
record.Document.Tenant,
record.IngestedAt,
record.CreatedAt,
record.Document))
.ToArray();
var response = new AdvisoryEvidenceResponse(recordResponses[0].Document.AdvisoryKey, recordResponses);
return JsonResult(response);
});
if (authorityConfigured)
{
advisoryEvidenceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
var aocVerifyEndpoint = app.MapPost("/aoc/verify", async (
HttpContext context,
AocVerifyRequest request,

View File

@@ -55,8 +55,8 @@
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | TODO | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
| CONCELIER-VULN-29-002 `Evidence retrieval API` | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | DONE (2025-11-07) | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
| CONCELIER-VULN-29-002 `Evidence retrieval API` | DOING (2025-11-07) | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
| CONCELIER-VULN-29-004 `Observability enhancements` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-VULN-29-001 | Instrument metrics/logs for observation + linkset pipelines (identifier collisions, withdrawn flags) and emit events consumed by Vuln Explorer resolver. |
## Advisory AI (Sprint 31)