Implement Advisory Canonicalization and Backfill Migration
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added AdvisoryCanonicalizer for canonicalizing advisory identifiers. - Created EnsureAdvisoryCanonicalKeyBackfillMigration to populate advisory_key and links in advisory_raw documents. - Introduced FileSurfaceManifestStore for managing surface manifests with file system backing. - Developed ISurfaceManifestReader and ISurfaceManifestWriter interfaces for reading and writing manifests. - Implemented SurfaceManifestPathBuilder for constructing paths and URIs for surface manifests. - Added tests for FileSurfaceManifestStore to ensure correct functionality and deterministic behavior. - Updated documentation for new features and migration steps.
This commit is contained in:
@@ -74,16 +74,20 @@ public sealed record AdvisoryRawRecordResponse(
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
|
||||
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
|
||||
|
||||
public sealed record AdvisoryRawListResponse(
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
|
||||
[property: JsonPropertyName("nextCursor")] string? NextCursor,
|
||||
[property: JsonPropertyName("hasMore")] bool HasMore);
|
||||
|
||||
public sealed record AdvisoryRawProvenanceResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
public sealed record AdvisoryRawListResponse(
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
|
||||
[property: JsonPropertyName("nextCursor")] string? NextCursor,
|
||||
[property: JsonPropertyName("hasMore")] bool HasMore);
|
||||
|
||||
public sealed record AdvisoryEvidenceResponse(
|
||||
[property: JsonPropertyName("advisoryKey")] string AdvisoryKey,
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records);
|
||||
|
||||
public sealed record AdvisoryRawProvenanceResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
|
||||
|
||||
@@ -61,24 +61,26 @@ internal static class AdvisoryRawRequestMapper
|
||||
identifiersRequest.Primary);
|
||||
|
||||
var linksetRequest = request.Linkset;
|
||||
var linkset = new RawLinkset
|
||||
{
|
||||
Aliases = NormalizeStrings(linksetRequest?.Aliases),
|
||||
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
|
||||
Cpes = NormalizeStrings(linksetRequest?.Cpes),
|
||||
References = NormalizeReferences(linksetRequest?.References),
|
||||
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
|
||||
Notes = NormalizeDictionary(linksetRequest?.Notes)
|
||||
};
|
||||
|
||||
return new AdvisoryRawDocument(
|
||||
tenant.Trim().ToLowerInvariant(),
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset);
|
||||
}
|
||||
var linkset = new RawLinkset
|
||||
{
|
||||
Aliases = NormalizeStrings(linksetRequest?.Aliases),
|
||||
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
|
||||
Cpes = NormalizeStrings(linksetRequest?.Cpes),
|
||||
References = NormalizeReferences(linksetRequest?.References),
|
||||
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
|
||||
Notes = NormalizeDictionary(linksetRequest?.Notes)
|
||||
};
|
||||
|
||||
return new AdvisoryRawDocument(
|
||||
tenant.Trim().ToLowerInvariant(),
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset,
|
||||
AdvisoryKey: string.Empty,
|
||||
Links: ImmutableArray<RawLink>.Empty);
|
||||
}
|
||||
|
||||
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
|
||||
{
|
||||
|
||||
@@ -144,7 +144,7 @@ public sealed class ConcelierOptions
|
||||
|
||||
public sealed class FeaturesOptions
|
||||
{
|
||||
public bool NoMergeEnabled { get; set; }
|
||||
public bool NoMergeEnabled { get; set; } = true;
|
||||
|
||||
public bool LnmShadowWrites { get; set; } = true;
|
||||
|
||||
|
||||
@@ -672,6 +672,59 @@ if (authorityConfigured)
|
||||
advisoryRawProvenanceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
|
||||
}
|
||||
|
||||
var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKey}", async (
|
||||
string advisoryKey,
|
||||
HttpContext context,
|
||||
[FromServices] IAdvisoryRawService rawService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
ApplyNoCache(context.Response);
|
||||
|
||||
if (!TryResolveTenant(context, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var authorizationError = EnsureTenantAuthorized(context, tenant);
|
||||
if (authorizationError is not null)
|
||||
{
|
||||
return authorizationError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(advisoryKey))
|
||||
{
|
||||
return Problem(context, "advisoryKey is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide an advisory identifier.");
|
||||
}
|
||||
|
||||
var vendorFilter = AdvisoryRawRequestMapper.NormalizeStrings(context.Request.Query["vendor"]);
|
||||
var records = await rawService.FindByAdvisoryKeyAsync(
|
||||
tenant,
|
||||
advisoryKey,
|
||||
vendorFilter,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var recordResponses = records
|
||||
.Select(record => new AdvisoryRawRecordResponse(
|
||||
record.Id,
|
||||
record.Document.Tenant,
|
||||
record.IngestedAt,
|
||||
record.CreatedAt,
|
||||
record.Document))
|
||||
.ToArray();
|
||||
|
||||
var response = new AdvisoryEvidenceResponse(recordResponses[0].Document.AdvisoryKey, recordResponses);
|
||||
return JsonResult(response);
|
||||
});
|
||||
if (authorityConfigured)
|
||||
{
|
||||
advisoryEvidenceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
|
||||
}
|
||||
|
||||
var aocVerifyEndpoint = app.MapPost("/aoc/verify", async (
|
||||
HttpContext context,
|
||||
AocVerifyRequest request,
|
||||
|
||||
@@ -55,8 +55,8 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | TODO | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
|
||||
| CONCELIER-VULN-29-002 `Evidence retrieval API` | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
|
||||
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | DONE (2025-11-07) | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
|
||||
| CONCELIER-VULN-29-002 `Evidence retrieval API` | DOING (2025-11-07) | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
|
||||
| CONCELIER-VULN-29-004 `Observability enhancements` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-VULN-29-001 | Instrument metrics/logs for observation + linkset pipelines (identifier collisions, withdrawn flags) and emit events consumed by Vuln Explorer resolver. |
|
||||
|
||||
## Advisory AI (Sprint 31)
|
||||
|
||||
Reference in New Issue
Block a user