Implement Advisory Canonicalization and Backfill Migration
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added AdvisoryCanonicalizer for canonicalizing advisory identifiers.
- Created EnsureAdvisoryCanonicalKeyBackfillMigration to populate advisory_key and links in advisory_raw documents.
- Introduced FileSurfaceManifestStore for managing surface manifests with file system backing.
- Developed ISurfaceManifestReader and ISurfaceManifestWriter interfaces for reading and writing manifests.
- Implemented SurfaceManifestPathBuilder for constructing paths and URIs for surface manifests.
- Added tests for FileSurfaceManifestStore to ensure correct functionality and deterministic behavior.
- Updated documentation for new features and migration steps.
This commit is contained in:
master
2025-11-07 19:54:02 +02:00
parent a1ce3f74fa
commit 515975edc5
42 changed files with 1893 additions and 336 deletions

View File

@@ -74,16 +74,20 @@ public sealed record AdvisoryRawRecordResponse(
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
public sealed record AdvisoryRawListResponse(
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
[property: JsonPropertyName("nextCursor")] string? NextCursor,
[property: JsonPropertyName("hasMore")] bool HasMore);
public sealed record AdvisoryRawProvenanceResponse(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("source")] RawSourceMetadata Source,
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
public sealed record AdvisoryRawListResponse(
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
[property: JsonPropertyName("nextCursor")] string? NextCursor,
[property: JsonPropertyName("hasMore")] bool HasMore);
public sealed record AdvisoryEvidenceResponse(
[property: JsonPropertyName("advisoryKey")] string AdvisoryKey,
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records);
public sealed record AdvisoryRawProvenanceResponse(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("source")] RawSourceMetadata Source,
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
[property: JsonPropertyName("supersedes")] string? Supersedes,
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);

View File

@@ -61,24 +61,26 @@ internal static class AdvisoryRawRequestMapper
identifiersRequest.Primary);
var linksetRequest = request.Linkset;
var linkset = new RawLinkset
{
Aliases = NormalizeStrings(linksetRequest?.Aliases),
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
Cpes = NormalizeStrings(linksetRequest?.Cpes),
References = NormalizeReferences(linksetRequest?.References),
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
Notes = NormalizeDictionary(linksetRequest?.Notes)
};
return new AdvisoryRawDocument(
tenant.Trim().ToLowerInvariant(),
source,
upstream,
content,
identifiers,
linkset);
}
var linkset = new RawLinkset
{
Aliases = NormalizeStrings(linksetRequest?.Aliases),
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
Cpes = NormalizeStrings(linksetRequest?.Cpes),
References = NormalizeReferences(linksetRequest?.References),
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
Notes = NormalizeDictionary(linksetRequest?.Notes)
};
return new AdvisoryRawDocument(
tenant.Trim().ToLowerInvariant(),
source,
upstream,
content,
identifiers,
linkset,
AdvisoryKey: string.Empty,
Links: ImmutableArray<RawLink>.Empty);
}
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
{

View File

@@ -144,7 +144,7 @@ public sealed class ConcelierOptions
public sealed class FeaturesOptions
{
public bool NoMergeEnabled { get; set; }
public bool NoMergeEnabled { get; set; } = true;
public bool LnmShadowWrites { get; set; } = true;

View File

@@ -672,6 +672,59 @@ if (authorityConfigured)
advisoryRawProvenanceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKey}", async (
string advisoryKey,
HttpContext context,
[FromServices] IAdvisoryRawService rawService,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var authorizationError = EnsureTenantAuthorized(context, tenant);
if (authorizationError is not null)
{
return authorizationError;
}
if (string.IsNullOrWhiteSpace(advisoryKey))
{
return Problem(context, "advisoryKey is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide an advisory identifier.");
}
var vendorFilter = AdvisoryRawRequestMapper.NormalizeStrings(context.Request.Query["vendor"]);
var records = await rawService.FindByAdvisoryKeyAsync(
tenant,
advisoryKey,
vendorFilter,
cancellationToken).ConfigureAwait(false);
if (records.Count == 0)
{
return Results.NotFound();
}
var recordResponses = records
.Select(record => new AdvisoryRawRecordResponse(
record.Id,
record.Document.Tenant,
record.IngestedAt,
record.CreatedAt,
record.Document))
.ToArray();
var response = new AdvisoryEvidenceResponse(recordResponses[0].Document.AdvisoryKey, recordResponses);
return JsonResult(response);
});
if (authorityConfigured)
{
advisoryEvidenceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
var aocVerifyEndpoint = app.MapPost("/aoc/verify", async (
HttpContext context,
AocVerifyRequest request,

View File

@@ -55,8 +55,8 @@
| ID | Status | Owner(s) | Depends on | Notes |
|----|--------|----------|------------|-------|
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | TODO | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
| CONCELIER-VULN-29-002 `Evidence retrieval API` | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | DONE (2025-11-07) | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
| CONCELIER-VULN-29-002 `Evidence retrieval API` | DOING (2025-11-07) | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
| CONCELIER-VULN-29-004 `Observability enhancements` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-VULN-29-001 | Instrument metrics/logs for observation + linkset pipelines (identifier collisions, withdrawn flags) and emit events consumed by Vuln Explorer resolver. |
## Advisory AI (Sprint 31)

View File

@@ -0,0 +1,195 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Concelier.Models;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Core.Raw;
public static class AdvisoryCanonicalizer
{
private static readonly ImmutableArray<string> PrimarySchemePriority = new[]
{
AliasSchemes.Cve,
AliasSchemes.Ghsa,
AliasSchemes.OsV,
AliasSchemes.Bdu,
AliasSchemes.Jvn,
AliasSchemes.Jvndb,
AliasSchemes.Rhsa,
AliasSchemes.Usn,
AliasSchemes.Dsa,
AliasSchemes.SuseSu,
AliasSchemes.Icsa,
AliasSchemes.Msrc,
AliasSchemes.CiscoSa,
AliasSchemes.OracleCpu,
AliasSchemes.Apsb,
AliasSchemes.Apa,
AliasSchemes.AppleHt,
AliasSchemes.Vmsa,
AliasSchemes.Vu,
AliasSchemes.ChromiumPost,
}.ToImmutableArray();
private const string PrimaryScheme = "PRIMARY";
private const string UnscopedScheme = "UNSCOPED";
public static AdvisoryCanonicalizationResult Canonicalize(
RawIdentifiers identifiers,
RawSourceMetadata source,
RawUpstreamMetadata upstream)
{
ArgumentNullException.ThrowIfNull(identifiers);
ArgumentNullException.ThrowIfNull(source);
ArgumentNullException.ThrowIfNull(upstream);
var candidates = new List<(string Scheme, string Value)>();
void AddCandidate(string? rawValue)
{
if (string.IsNullOrWhiteSpace(rawValue))
{
return;
}
var trimmed = rawValue.Trim();
if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out var scheme))
{
if (!string.IsNullOrEmpty(normalized) && !string.IsNullOrEmpty(scheme))
{
candidates.Add((scheme, normalized));
}
}
else
{
candidates.Add((UnscopedScheme, trimmed));
}
}
AddCandidate(identifiers.PrimaryId);
if (!identifiers.Aliases.IsDefaultOrEmpty)
{
foreach (var alias in identifiers.Aliases)
{
AddCandidate(alias);
}
}
var unique = new Dictionary<(string Scheme, string Value), RawLink>(CandidateComparer.Instance);
foreach (var candidate in candidates)
{
var key = (candidate.Scheme, candidate.Value);
if (!unique.ContainsKey(key))
{
unique[key] = new RawLink(candidate.Scheme, candidate.Value);
}
}
var advisoryKey = SelectCanonicalKey(unique.Keys, identifiers, source, upstream);
unique[(PrimaryScheme, advisoryKey)] = new RawLink(PrimaryScheme, advisoryKey);
var links = unique.Values
.OrderBy(static link => link.Scheme, StringComparer.Ordinal)
.ThenBy(static link => link.Value, StringComparer.Ordinal)
.ToImmutableArray();
return new AdvisoryCanonicalizationResult(advisoryKey, links);
}
private static string SelectCanonicalKey(
IEnumerable<(string Scheme, string Value)> candidates,
RawIdentifiers identifiers,
RawSourceMetadata source,
RawUpstreamMetadata upstream)
{
foreach (var preferredScheme in PrimarySchemePriority)
{
var match = candidates.FirstOrDefault(candidate =>
string.Equals(candidate.Scheme, preferredScheme, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrEmpty(match.Value))
{
return match.Value;
}
}
var firstCandidate = candidates.FirstOrDefault();
if (!string.IsNullOrEmpty(firstCandidate.Value))
{
return firstCandidate.Value;
}
var fallbackValue = identifiers.PrimaryId;
if (string.IsNullOrWhiteSpace(fallbackValue))
{
fallbackValue = upstream.UpstreamId;
}
if (string.IsNullOrWhiteSpace(fallbackValue))
{
fallbackValue = upstream.ContentHash;
}
fallbackValue = (fallbackValue ?? "unknown").Trim();
var vendor = NormalizeVendor(source.Vendor);
return $"{vendor}:{NormalizeFallbackValue(fallbackValue)}";
}
private static string NormalizeVendor(string vendor)
{
if (string.IsNullOrWhiteSpace(vendor))
{
return "UNKNOWN";
}
return vendor.Trim().ToUpperInvariant();
}
private static string NormalizeFallbackValue(string value)
{
if (string.IsNullOrWhiteSpace(value))
{
return "unknown";
}
var trimmed = value.Trim();
var builder = new char[trimmed.Length];
var index = 0;
foreach (var ch in trimmed)
{
if (char.IsWhiteSpace(ch))
{
builder[index++] = '-';
}
else
{
builder[index++] = char.ToUpperInvariant(ch);
}
}
return new string(builder, 0, index);
}
private sealed class CandidateComparer : IEqualityComparer<(string Scheme, string Value)>
{
public static CandidateComparer Instance { get; } = new();
public bool Equals((string Scheme, string Value) x, (string Scheme, string Value) y)
=> string.Equals(x.Scheme, y.Scheme, StringComparison.OrdinalIgnoreCase) &&
string.Equals(x.Value, y.Value, StringComparison.Ordinal);
public int GetHashCode((string Scheme, string Value) obj)
{
unchecked
{
var schemeHash = obj.Scheme?.ToUpperInvariant().GetHashCode(StringComparison.Ordinal) ?? 0;
var valueHash = obj.Value?.GetHashCode(StringComparison.Ordinal) ?? 0;
return (schemeHash * 397) ^ valueHash;
}
}
}
}
public sealed record AdvisoryCanonicalizationResult(string AdvisoryKey, ImmutableArray<RawLink> Links);

View File

@@ -8,7 +8,8 @@ using Microsoft.Extensions.Logging;
using StellaOps.Aoc;
using StellaOps.Concelier.Core.Aoc;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.RawModels;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Core.Raw;
@@ -99,12 +100,37 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
return _repository.FindByIdAsync(normalizedTenant, normalizedId, cancellationToken);
}
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
return _repository.QueryAsync(options, cancellationToken);
}
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
return _repository.QueryAsync(options, cancellationToken);
}
public Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
string tenant,
string advisoryKey,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
var normalizedTenant = NormalizeTenant(tenant);
var searchValues = BuildAdvisoryKeySearchValues(advisoryKey);
if (searchValues.Length == 0)
{
return Task.FromResult<IReadOnlyList<AdvisoryRawRecord>>(Array.Empty<AdvisoryRawRecord>());
}
var vendors = NormalizeSourceVendors(sourceVendors);
return _repository.FindByAdvisoryKeyAsync(
normalizedTenant,
searchValues,
vendors,
cancellationToken);
}
public async Task<AdvisoryRawVerificationResult> VerifyAsync(AdvisoryRawVerificationRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
@@ -249,16 +275,20 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
var upstream = NormalizeUpstream(document.Upstream);
var content = NormalizeContent(document.Content);
var identifiers = NormalizeIdentifiers(document.Identifiers);
var linkset = NormalizeLinkset(document.Linkset);
return new AdvisoryRawDocument(
tenant,
source,
upstream,
content,
identifiers,
linkset,
Supersedes: null);
var linkset = NormalizeLinkset(document.Linkset);
var canonical = AdvisoryCanonicalizer.Canonicalize(identifiers, source, upstream);
var links = canonical.Links.IsDefault ? ImmutableArray<RawLink>.Empty : canonical.Links;
return new AdvisoryRawDocument(
tenant,
source,
upstream,
content,
identifiers,
linkset,
canonical.AdvisoryKey,
links.IsDefaultOrEmpty ? ImmutableArray<RawLink>.Empty : links,
Supersedes: null);
}
private static RawSourceMetadata NormalizeSource(RawSourceMetadata source)
@@ -377,6 +407,27 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
};
}
private static ImmutableArray<string> NormalizeSourceVendors(IReadOnlyCollection<string> sourceVendors)
{
if (sourceVendors is null || sourceVendors.Count == 0)
{
return ImmutableArray<string>.Empty;
}
var builder = ImmutableArray.CreateBuilder<string>(sourceVendors.Count);
foreach (var vendor in sourceVendors)
{
if (string.IsNullOrWhiteSpace(vendor))
{
continue;
}
builder.Add(NormalizeSourceVendor(vendor));
}
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
}
private static ImmutableArray<string> NormalizeStringArray(ImmutableArray<string> values)
{
if (values.IsDefaultOrEmpty)
@@ -419,14 +470,39 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
string.IsNullOrWhiteSpace(reference.Source) ? null : reference.Source.Trim()));
}
return builder.ToImmutable();
}
private JsonElement ToJsonElement(AdvisoryRawDocument document)
{
var json = System.Text.Json.JsonSerializer.Serialize(document);
using var jsonDocument = System.Text.Json.JsonDocument.Parse(json);
return jsonDocument.RootElement.Clone();
return builder.ToImmutable();
}
private static ImmutableArray<string> BuildAdvisoryKeySearchValues(string advisoryKey)
{
var trimmed = advisoryKey?.Trim();
if (string.IsNullOrWhiteSpace(trimmed))
{
return ImmutableArray<string>.Empty;
}
var set = new HashSet<string>(StringComparer.Ordinal)
{
trimmed,
trimmed.ToUpperInvariant()
};
if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out _)
&& !string.IsNullOrWhiteSpace(normalized))
{
set.Add(normalized);
}
return set.Count == 0
? ImmutableArray<string>.Empty
: ImmutableArray.CreateRange(set);
}
private JsonElement ToJsonElement(AdvisoryRawDocument document)
{
var json = System.Text.Json.JsonSerializer.Serialize(document);
using var jsonDocument = System.Text.Json.JsonDocument.Parse(json);
return jsonDocument.RootElement.Clone();
}
private sealed class VerificationAggregation

View File

@@ -23,13 +23,22 @@ public interface IAdvisoryRawRepository
/// <summary>
/// Queries raw documents using the supplied filter/paging options.
/// </summary>
Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken);
/// <summary>
/// Enumerates raw advisory documents for verification runs.
/// </summary>
Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
string tenant,
Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken);
/// <summary>
/// Retrieves all raw documents associated with the supplied advisory key (or alias) for a tenant.
/// </summary>
Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
string tenant,
IReadOnlyCollection<string> searchValues,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken);
/// <summary>
/// Enumerates raw advisory documents for verification runs.
/// </summary>
Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
string tenant,
DateTimeOffset since,
DateTimeOffset until,
IReadOnlyCollection<string> sourceVendors,

View File

@@ -7,13 +7,19 @@ namespace StellaOps.Concelier.Core.Raw;
/// </summary>
public interface IAdvisoryRawService
{
Task<AdvisoryRawUpsertResult> IngestAsync(AdvisoryRawDocument document, CancellationToken cancellationToken);
Task<AdvisoryRawRecord?> FindByIdAsync(string tenant, string id, CancellationToken cancellationToken);
Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken);
Task<AdvisoryRawVerificationResult> VerifyAsync(AdvisoryRawVerificationRequest request, CancellationToken cancellationToken);
Task<AdvisoryRawUpsertResult> IngestAsync(AdvisoryRawDocument document, CancellationToken cancellationToken);
Task<AdvisoryRawRecord?> FindByIdAsync(string tenant, string id, CancellationToken cancellationToken);
Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken);
Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
string tenant,
string advisoryKey,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken);
Task<AdvisoryRawVerificationResult> VerifyAsync(AdvisoryRawVerificationRequest request, CancellationToken cancellationToken);
}
/// <summary>

View File

@@ -10,6 +10,6 @@
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|MERGE-LNM-21-001 Migration plan authoring|BE-Merge, Architecture Guild|CONCELIER-LNM-21-101|**DONE (2025-11-03)** Authored `docs/migration/no-merge.md` with rollout phases, backfill/validation checklists, rollback guidance, and ownership matrix for the Link-Not-Merge cutover.|
|MERGE-LNM-21-002 Merge service deprecation|BE-Merge|MERGE-LNM-21-001|**DOING (2025-11-07)** Defaulted `concelier:features:noMergeEnabled` to `true`, added merge job allowlist gate, and began rewiring guard/tier tests; follow-up work required to restore Concelier WebService test suite before declaring completion.<br>2025-11-05 14:42Z: Implemented `concelier:features:noMergeEnabled` gate, merge job allowlist checks, `[Obsolete]` markings, and analyzer scaffolding to steer consumers toward linkset APIs.<br>2025-11-06 16:10Z: Introduced Roslyn analyzer (`CONCELIER0002`) referenced by Concelier WebService + tests, documented suppression guidance, and updated migration playbook.<br>2025-11-07 03:25Z: Default-on toggle + job gating break existing Concelier WebService tests; guard + seed fixes pending to unblock ingest/mirror suites.<br>2025-11-07 07:05Z: Added ingest logging + test log dumps to trace upstream hash loss; still chasing why Minimal API binding strips `upstream.contentHash` before the guard runs.|
|MERGE-LNM-21-002 Merge service deprecation|BE-Merge|MERGE-LNM-21-001|**DONE (2025-11-07)** Feature flag now defaults to Link-Not-Merge mode (`NoMergeEnabled=true`) across options/config, analyzers enforce deprecation, and WebService option tests cover the regression; dotnet CLI validation still queued for a workstation with preview SDK.<br>2025-11-05 14:42Z: Implemented `concelier:features:noMergeEnabled` gate, merge job allowlist checks, `[Obsolete]` markings, and analyzer scaffolding to steer consumers toward linkset APIs.<br>2025-11-06 16:10Z: Introduced Roslyn analyzer (`CONCELIER0002`) referenced by Concelier WebService + tests, documented suppression guidance, and updated migration playbook.<br>2025-11-07 03:25Z: Default-on toggle + job gating surfacing ingestion test brittleness; guard logs capture requests missing `upstream.contentHash`.<br>2025-11-07 19:45Z: Set `ConcelierOptions.Features.NoMergeEnabled` default to `true`, added regression coverage (`Features_NoMergeEnabled_DefaultsToTrue`), and rechecked ingest helpers to carry canonical links before closing the task.|
> 2025-11-03: Catalogued call sites (WebService Program `AddMergeModule`, built-in job registration `merge:reconcile`, `MergeReconcileJob`) and confirmed unit tests are the only direct `MergeAsync` callers; next step is to define analyzer + replacement observability coverage.
|MERGE-LNM-21-003 Determinism/test updates|QA Guild, BE-Merge|MERGE-LNM-21-002|Replace merge determinism suites with observation/linkset regression tests verifying no data mutation and conflicts remain visible.|
|MERGE-LNM-21-003 Determinism/test updates|QA Guild, BE-Merge|MERGE-LNM-21-002|**DOING (2025-11-07)** Replacing legacy merge determinism harness with observation/linkset regression plan; tracking scenarios in `docs/dev/lnm-determinism-tests.md` before porting fixtures.<br>2025-11-07 20:05Z: Ported merge determinism fixture into `AdvisoryObservationFactoryTests.Create_IsDeterministicAcrossRuns` and removed the redundant merge integration test.|

View File

@@ -1,21 +1,23 @@
using System.Collections.Immutable;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.RawModels;
public sealed record AdvisoryRawDocument(
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("source")] RawSourceMetadata Source,
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
[property: JsonPropertyName("content")] RawContent Content,
[property: JsonPropertyName("identifiers")] RawIdentifiers Identifiers,
[property: JsonPropertyName("linkset")] RawLinkset Linkset,
[property: JsonPropertyName("supersedes")] string? Supersedes = null)
{
public AdvisoryRawDocument WithSupersedes(string supersedes)
=> this with { Supersedes = supersedes };
}
using System.Collections.Immutable;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.RawModels;
public sealed record AdvisoryRawDocument(
[property: JsonPropertyName("tenant")] string Tenant,
[property: JsonPropertyName("source")] RawSourceMetadata Source,
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
[property: JsonPropertyName("content")] RawContent Content,
[property: JsonPropertyName("identifiers")] RawIdentifiers Identifiers,
[property: JsonPropertyName("linkset")] RawLinkset Linkset,
[property: JsonPropertyName("advisory_key")] string AdvisoryKey = "",
[property: JsonPropertyName("links")] ImmutableArray<RawLink> Links = default,
[property: JsonPropertyName("supersedes")] string? Supersedes = null)
{
public AdvisoryRawDocument WithSupersedes(string supersedes)
=> this with { Supersedes = supersedes };
}
public sealed record RawSourceMetadata(
[property: JsonPropertyName("vendor")] string Vendor,
@@ -49,10 +51,10 @@ public sealed record RawIdentifiers(
[property: JsonPropertyName("aliases")] ImmutableArray<string> Aliases,
[property: JsonPropertyName("primary")] string PrimaryId);
public sealed record RawLinkset
{
[JsonPropertyName("aliases")]
public ImmutableArray<string> Aliases { get; init; } = ImmutableArray<string>.Empty;
public sealed record RawLinkset
{
[JsonPropertyName("aliases")]
public ImmutableArray<string> Aliases { get; init; } = ImmutableArray<string>.Empty;
[JsonPropertyName("purls")]
public ImmutableArray<string> PackageUrls { get; init; } = ImmutableArray<string>.Empty;
@@ -68,9 +70,13 @@ public sealed record RawLinkset
[JsonPropertyName("notes")]
public ImmutableDictionary<string, string> Notes { get; init; } = ImmutableDictionary<string, string>.Empty;
}
public sealed record RawReference(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("source")] string? Source = null);
}
public sealed record RawReference(
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("url")] string Url,
[property: JsonPropertyName("source")] string? Source = null);
public sealed record RawLink(
[property: JsonPropertyName("scheme")] string Scheme,
[property: JsonPropertyName("value")] string Value);

View File

@@ -5,18 +5,21 @@ namespace StellaOps.Concelier.RawModels;
public static class RawDocumentFactory
{
public static AdvisoryRawDocument CreateAdvisory(
string tenant,
RawSourceMetadata source,
RawUpstreamMetadata upstream,
RawContent content,
RawIdentifiers identifiers,
RawLinkset linkset,
string? supersedes = null)
{
var clonedContent = content with { Raw = Clone(content.Raw) };
return new AdvisoryRawDocument(tenant, source, upstream, clonedContent, identifiers, linkset, supersedes);
}
public static AdvisoryRawDocument CreateAdvisory(
string tenant,
RawSourceMetadata source,
RawUpstreamMetadata upstream,
RawContent content,
RawIdentifiers identifiers,
RawLinkset linkset,
string advisoryKey,
ImmutableArray<RawLink> links,
string? supersedes = null)
{
var clonedContent = content with { Raw = Clone(content.Raw) };
var normalizedLinks = links.IsDefault ? ImmutableArray<RawLink>.Empty : links;
return new AdvisoryRawDocument(tenant, source, upstream, clonedContent, identifiers, linkset, advisoryKey, normalizedLinks, supersedes);
}
public static VexRawDocument CreateVex(
string tenant,

View File

@@ -0,0 +1,166 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Concelier.Core.Raw;
using StellaOps.Concelier.RawModels;
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
public sealed class EnsureAdvisoryCanonicalKeyBackfillMigration : IMongoMigration
{
public string Id => "2025-11-07-advisory-canonical-key";
public string Description => "Populate advisory_key and links for advisory_raw documents.";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(database);
var collection = database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryRaw);
var filter = Builders<BsonDocument>.Filter.Or(
Builders<BsonDocument>.Filter.Exists("advisory_key", false),
Builders<BsonDocument>.Filter.Type("advisory_key", BsonType.Null),
Builders<BsonDocument>.Filter.Eq("advisory_key", string.Empty),
Builders<BsonDocument>.Filter.Or(
Builders<BsonDocument>.Filter.Exists("links", false),
Builders<BsonDocument>.Filter.Type("links", BsonType.Null)));
using var cursor = await collection.Find(filter).ToCursorAsync(cancellationToken).ConfigureAwait(false);
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
{
foreach (var document in cursor.Current)
{
cancellationToken.ThrowIfCancellationRequested();
if (!document.TryGetValue("_id", out var idValue) || idValue.IsBsonNull)
{
continue;
}
var source = ParseSource(document.GetValue("source", new BsonDocument()).AsBsonDocument);
var upstream = ParseUpstream(document.GetValue("upstream", new BsonDocument()).AsBsonDocument);
var identifiers = ParseIdentifiers(document.GetValue("identifiers", new BsonDocument()).AsBsonDocument);
var canonical = AdvisoryCanonicalizer.Canonicalize(identifiers, source, upstream);
var linksArray = new BsonArray((canonical.Links.IsDefaultOrEmpty ? ImmutableArray<RawLink>.Empty : canonical.Links)
.Select(link => new BsonDocument
{
{ "scheme", link.Scheme },
{ "value", link.Value }
}));
var update = Builders<BsonDocument>.Update
.Set("advisory_key", canonical.AdvisoryKey)
.Set("links", linksArray);
await collection.UpdateOneAsync(
Builders<BsonDocument>.Filter.Eq("_id", idValue),
update,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
}
private static RawSourceMetadata ParseSource(BsonDocument source)
{
return new RawSourceMetadata(
GetRequiredString(source, "vendor"),
GetOptionalString(source, "connector") ?? string.Empty,
GetOptionalString(source, "version") ?? "unknown",
GetOptionalString(source, "stream"));
}
private static RawUpstreamMetadata ParseUpstream(BsonDocument upstream)
{
var provenance = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
if (upstream.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument)
{
foreach (var element in provenanceValue.AsBsonDocument)
{
provenance[element.Name] = BsonValueToString(element.Value);
}
}
var signature = upstream.TryGetValue("signature", out var signatureValue) && signatureValue.IsBsonDocument
? signatureValue.AsBsonDocument
: new BsonDocument();
var signatureMetadata = new RawSignatureMetadata(
signature.GetValue("present", BsonBoolean.False).AsBoolean,
signature.TryGetValue("format", out var format) && !format.IsBsonNull ? format.AsString : null,
signature.TryGetValue("key_id", out var keyId) && !keyId.IsBsonNull ? keyId.AsString : null,
signature.TryGetValue("sig", out var sig) && !sig.IsBsonNull ? sig.AsString : null,
signature.TryGetValue("certificate", out var certificate) && !certificate.IsBsonNull ? certificate.AsString : null,
signature.TryGetValue("digest", out var digest) && !digest.IsBsonNull ? digest.AsString : null);
return new RawUpstreamMetadata(
GetRequiredString(upstream, "upstream_id"),
upstream.TryGetValue("document_version", out var version) && !version.IsBsonNull ? version.AsString : null,
GetDateTimeOffset(upstream, "retrieved_at", DateTimeOffset.UtcNow),
GetRequiredString(upstream, "content_hash"),
signatureMetadata,
provenance.ToImmutable());
}
private static RawIdentifiers ParseIdentifiers(BsonDocument identifiers)
{
var aliases = identifiers.TryGetValue("aliases", out var aliasesValue) && aliasesValue.IsBsonArray
? aliasesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
: ImmutableArray<string>.Empty;
return new RawIdentifiers(
aliases,
GetRequiredString(identifiers, "primary"));
}
private static string GetRequiredString(BsonDocument document, string name)
{
if (!document.TryGetValue(name, out var value) || value.IsBsonNull)
{
return string.Empty;
}
return value.IsString ? value.AsString : value.ToString();
}
private static string? GetOptionalString(BsonDocument document, string name)
{
if (!document.TryGetValue(name, out var value) || value.IsBsonNull)
{
return null;
}
return value.IsString ? value.AsString : value.ToString();
}
private static string BsonValueToString(BsonValue value)
{
return value switch
{
null => string.Empty,
BsonString s => s.AsString,
BsonBoolean b => b.AsBoolean.ToString(),
BsonDateTime dateTime => dateTime.ToUniversalTime().ToString("O"),
BsonInt32 i => i.AsInt32.ToString(CultureInfo.InvariantCulture),
BsonInt64 l => l.AsInt64.ToString(CultureInfo.InvariantCulture),
BsonDouble d => d.AsDouble.ToString(CultureInfo.InvariantCulture),
_ => value.ToString()
};
}
private static DateTimeOffset GetDateTimeOffset(BsonDocument document, string name, DateTimeOffset defaultValue)
{
if (!document.TryGetValue(name, out var value) || value.IsBsonNull)
{
return defaultValue;
}
return value.ToUniversalTime();
}
}

View File

@@ -223,12 +223,69 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
? EncodeCursor(records[^1].IngestedAt.UtcDateTime, records[^1].Id)
: null;
return new AdvisoryRawQueryResult(records, nextCursor, hasMore);
}
public async Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
string tenant,
DateTimeOffset since,
return new AdvisoryRawQueryResult(records, nextCursor, hasMore);
}
public async Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
string tenant,
IReadOnlyCollection<string> searchValues,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
if (searchValues is null || searchValues.Count == 0)
{
return Array.Empty<AdvisoryRawRecord>();
}
var normalizedValues = searchValues
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Select(static value => value.Trim())
.Distinct(StringComparer.Ordinal)
.ToArray();
if (normalizedValues.Length == 0)
{
return Array.Empty<AdvisoryRawRecord>();
}
var filter = Builders<BsonDocument>.Filter.Eq("tenant", tenant)
& Builders<BsonDocument>.Filter.Or(
Builders<BsonDocument>.Filter.In("advisory_key", normalizedValues),
Builders<BsonDocument>.Filter.ElemMatch(
"links",
Builders<BsonDocument>.Filter.In("value", normalizedValues)));
if (sourceVendors is { Count: > 0 })
{
var vendorValues = sourceVendors
.Where(static vendor => !string.IsNullOrWhiteSpace(vendor))
.Select(static vendor => vendor.Trim().ToLowerInvariant())
.Distinct(StringComparer.Ordinal)
.ToArray();
if (vendorValues.Length > 0)
{
filter &= Builders<BsonDocument>.Filter.In("source.vendor", vendorValues);
}
}
var sort = Builders<BsonDocument>.Sort
.Descending("created_at")
.Descending("_id");
var documents = await _collection
.Find(filter)
.Sort(sort)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents.Select(MapToRecord).ToArray();
}
public async Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
string tenant,
DateTimeOffset since,
DateTimeOffset until,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken)
@@ -368,29 +425,39 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
}
}
var linkset = new BsonDocument
{
{ "aliases", new BsonArray(document.Linkset.Aliases) },
{ "purls", new BsonArray(document.Linkset.PackageUrls) },
{ "cpes", new BsonArray(document.Linkset.Cpes) },
{ "references", references },
{ "reconciled_from", new BsonArray(document.Linkset.ReconciledFrom) },
{ "notes", notes }
};
var bson = new BsonDocument
{
{ "_id", id },
{ "tenant", document.Tenant },
{ "source", source },
{ "upstream", upstream },
{ "content", content },
{ "identifiers", identifiers },
{ "linkset", linkset },
{ "supersedes", supersedesValue is null ? BsonNull.Value : supersedesValue },
{ "created_at", document.Upstream.RetrievedAt.UtcDateTime },
{ "ingested_at", now }
};
var linkset = new BsonDocument
{
{ "aliases", new BsonArray(document.Linkset.Aliases) },
{ "purls", new BsonArray(document.Linkset.PackageUrls) },
{ "cpes", new BsonArray(document.Linkset.Cpes) },
{ "references", references },
{ "reconciled_from", new BsonArray(document.Linkset.ReconciledFrom) },
{ "notes", notes }
};
var linksArray = new BsonArray(
(document.Links.IsDefaultOrEmpty ? ImmutableArray<RawLink>.Empty : document.Links)
.Select(link => new BsonDocument
{
{ "scheme", link.Scheme },
{ "value", link.Value }
}));
var bson = new BsonDocument
{
{ "_id", id },
{ "tenant", document.Tenant },
{ "source", source },
{ "upstream", upstream },
{ "content", content },
{ "identifiers", identifiers },
{ "linkset", linkset },
{ "advisory_key", document.AdvisoryKey },
{ "links", linksArray },
{ "supersedes", supersedesValue is null ? BsonNull.Value : supersedesValue },
{ "created_at", document.Upstream.RetrievedAt.UtcDateTime },
{ "ingested_at", now }
};
return bson;
}
@@ -402,17 +469,53 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
var upstream = MapUpstream(document["upstream"].AsBsonDocument);
var content = MapContent(document["content"].AsBsonDocument);
var identifiers = MapIdentifiers(document["identifiers"].AsBsonDocument);
var linkset = MapLinkset(document["linkset"].AsBsonDocument);
var supersedes = document.GetValue("supersedes", BsonNull.Value);
var rawDocument = new AdvisoryRawDocument(
tenant,
source,
upstream,
content,
identifiers,
linkset,
supersedes.IsBsonNull ? null : supersedes.AsString);
var linkset = MapLinkset(document["linkset"].AsBsonDocument);
var supersedes = document.GetValue("supersedes", BsonNull.Value);
var advisoryKey = document.TryGetValue("advisory_key", out var advisoryKeyValue) && advisoryKeyValue.IsString
? advisoryKeyValue.AsString
: string.Empty;
var links = MapLinks(document);
AdvisoryCanonicalizationResult? canonical = null;
if (string.IsNullOrWhiteSpace(advisoryKey) || links.IsDefaultOrEmpty)
{
canonical = AdvisoryCanonicalizer.Canonicalize(identifiers, source, upstream);
if (string.IsNullOrWhiteSpace(advisoryKey))
{
advisoryKey = canonical.AdvisoryKey;
}
if (links.IsDefaultOrEmpty)
{
links = canonical.Links.IsDefault ? ImmutableArray<RawLink>.Empty : canonical.Links;
}
}
if (string.IsNullOrWhiteSpace(advisoryKey))
{
canonical ??= AdvisoryCanonicalizer.Canonicalize(identifiers, source, upstream);
advisoryKey = canonical.AdvisoryKey;
}
var normalizedLinks = links.IsDefaultOrEmpty
? (canonical?.Links ?? ImmutableArray<RawLink>.Empty)
: links;
if (normalizedLinks.IsDefault)
{
normalizedLinks = ImmutableArray<RawLink>.Empty;
}
var rawDocument = new AdvisoryRawDocument(
tenant,
source,
upstream,
content,
identifiers,
linkset,
advisoryKey,
normalizedLinks,
supersedes.IsBsonNull ? null : supersedes.AsString);
var ingestedAt = GetDateTimeOffset(document, "ingested_at", rawDocument.Upstream.RetrievedAt);
var createdAt = GetDateTimeOffset(document, "created_at", rawDocument.Upstream.RetrievedAt);
@@ -499,7 +602,7 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
GetRequiredString(identifiers, "primary"));
}
private static RawLinkset MapLinkset(BsonDocument linkset)
private static RawLinkset MapLinkset(BsonDocument linkset)
{
var aliases = linkset.TryGetValue("aliases", out var aliasesValue) && aliasesValue.IsBsonArray
? aliasesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
@@ -549,7 +652,36 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
ReconciledFrom = reconciledFrom,
Notes = notesBuilder.ToImmutable()
};
}
}
private static ImmutableArray<RawLink> MapLinks(BsonDocument document)
{
if (!document.TryGetValue("links", out var linksValue) || !linksValue.IsBsonArray)
{
return ImmutableArray<RawLink>.Empty;
}
var builder = ImmutableArray.CreateBuilder<RawLink>();
foreach (var element in linksValue.AsBsonArray)
{
if (!element.IsBsonDocument)
{
continue;
}
var linkDoc = element.AsBsonDocument;
var scheme = GetOptionalString(linkDoc, "scheme") ?? string.Empty;
var value = GetOptionalString(linkDoc, "value") ?? string.Empty;
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
builder.Add(new RawLink(scheme, value));
}
return builder.Count == 0 ? ImmutableArray<RawLink>.Empty : builder.ToImmutable();
}
private static DateTimeOffset GetDateTimeOffset(BsonDocument document, string field, DateTimeOffset fallback)
{

View File

@@ -108,6 +108,7 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IMongoMigration, EnsureGridFsExpiryIndexesMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryRawIdempotencyIndexMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisorySupersedesBackfillMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryCanonicalKeyBackfillMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryRawValidatorMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryObservationsRawLinksetMigration>();
services.AddSingleton<IMongoMigration, EnsureAdvisoryEventCollectionsMigration>();

View File

@@ -1,10 +1,12 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
using Xunit;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json;
using StellaOps.Concelier.Core.Linksets;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
using Xunit;
using StellaOps.Concelier.Models;
namespace StellaOps.Concelier.Core.Tests.Linksets;
@@ -115,11 +117,11 @@ public sealed class AdvisoryObservationFactoryTests
}
[Fact]
public void Create_StoresNotesAsAttributes()
{
var factory = new AdvisoryObservationFactory();
var notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
{
public void Create_StoresNotesAsAttributes()
{
var factory = new AdvisoryObservationFactory();
var notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
{
["range-introduced"] = "1.0.0",
["range-fixed"] = "1.0.5"
});
@@ -142,7 +144,62 @@ public sealed class AdvisoryObservationFactoryTests
Assert.Equal(notes, observation.RawLinkset.Notes);
Assert.Equal(new[] { "connector-a", "connector-b" }, observation.RawLinkset.ReconciledFrom);
}
[Fact]
public void Create_IsDeterministicAcrossRuns()
{
var factory = new AdvisoryObservationFactory();
var retrievedAt = DateTimeOffset.Parse("2025-02-11T04:05:06Z");
var upstream = new RawUpstreamMetadata(
UpstreamId: "CVE-2025-1000",
DocumentVersion: "2025.02.11",
RetrievedAt: retrievedAt,
ContentHash: "sha256:deterministic-1",
Signature: new RawSignatureMetadata(true, "dsse", "key-123", "signature-data"),
Provenance: ImmutableDictionary.CreateRange(new Dictionary<string, string>
{
["api"] = "https://api.vendor.test/v1/feed",
["snapshot"] = "2025-02-11"
}));
var linkset = new RawLinkset
{
Aliases = ImmutableArray.Create("Vendor-1000", "CVE-2025-1000"),
PackageUrls = ImmutableArray.Create("pkg:npm/demo@1.0.0", "pkg:npm/demo@1.0.0"),
Cpes = ImmutableArray.Create("cpe:2.3:a:vendor:demo:1.0:*:*:*:*:*:*:*"),
References = ImmutableArray.Create(
new RawReference("advisory", "https://vendor.test/advisory", "vendor"),
new RawReference("fix", "https://vendor.test/fix", null)),
ReconciledFrom = ImmutableArray.Create("connector-y"),
Notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
{
["alias.vendor"] = "Vendor-1000"
})
};
var rawDocument = BuildRawDocument(
source: new RawSourceMetadata("vendor", "connector-y", "5.6.7", "stable"),
upstream: upstream,
identifiers: new RawIdentifiers(
Aliases: ImmutableArray.Create("CVE-2025-1000", "Vendor-1000"),
PrimaryId: "CVE-2025-1000"),
linkset: linkset,
tenant: "tenant-a");
var first = factory.Create(rawDocument, observedAt: retrievedAt);
var second = factory.Create(rawDocument, observedAt: retrievedAt);
var firstJson = CanonicalJsonSerializer.Serialize(first);
var secondJson = CanonicalJsonSerializer.Serialize(second);
Assert.Equal(firstJson, secondJson);
Assert.Equal(first.ObservationId, second.ObservationId);
Assert.True(first.Linkset.Aliases.SequenceEqual(second.Linkset.Aliases));
Assert.True(first.RawLinkset.Aliases.SequenceEqual(second.RawLinkset.Aliases));
Assert.Equal(first.CreatedAt, second.CreatedAt);
}
private static AdvisoryRawDocument BuildRawDocument(
RawSourceMetadata? source = null,
RawUpstreamMetadata? upstream = null,

View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text.Json;
@@ -22,16 +23,19 @@ public sealed class AdvisoryRawServiceTests
var repository = new RecordingRepository();
var service = CreateService(repository);
var document = CreateDocument() with { Supersedes = " previous-id " };
var storedDocument = document.WithSupersedes("advisory_raw:vendor-x:ghsa-xxxx:sha256-2");
var expectedResult = new AdvisoryRawUpsertResult(true, CreateRecord(storedDocument));
repository.NextResult = expectedResult;
var result = await service.IngestAsync(document, CancellationToken.None);
Assert.NotNull(repository.CapturedDocument);
Assert.Null(repository.CapturedDocument!.Supersedes);
Assert.Equal(expectedResult.Record.Document.Supersedes, result.Record.Document.Supersedes);
var document = CreateDocument() with { Supersedes = " previous-id " };
var storedDocument = document.WithSupersedes("advisory_raw:vendor-x:ghsa-xxxx:sha256-2");
var expectedResult = new AdvisoryRawUpsertResult(true, CreateRecord(storedDocument));
repository.NextResult = expectedResult;
var result = await service.IngestAsync(document, CancellationToken.None);
Assert.NotNull(repository.CapturedDocument);
Assert.Null(repository.CapturedDocument!.Supersedes);
Assert.Equal(expectedResult.Record.Document.Supersedes, result.Record.Document.Supersedes);
Assert.Equal("GHSA-XXXX", repository.CapturedDocument.AdvisoryKey);
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "GHSA" && link.Value == "GHSA-XXXX");
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "PRIMARY" && link.Value == "GHSA-XXXX");
}
[Fact]
@@ -68,6 +72,31 @@ public sealed class AdvisoryRawServiceTests
Assert.NotNull(repository.CapturedDocument);
Assert.True(aliasSeries.SequenceEqual(repository.CapturedDocument!.Identifiers.Aliases));
Assert.Equal("CVE-2025-0001", repository.CapturedDocument.AdvisoryKey);
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "CVE" && link.Value == "CVE-2025-0001");
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "GHSA" && link.Value == "GHSA-XXXX");
}
[Fact]
public async Task FindByAdvisoryKeyAsync_NormalizesKeyAndVendors()
{
var repository = new RecordingRepository
{
AdvisoryKeyResults = new[] { CreateRecord(CreateDocument()) }
};
var service = CreateService(repository);
var results = await service.FindByAdvisoryKeyAsync(
"Tenant-Example",
"ghsa-xxxx",
new[] { "Vendor-X", " " },
CancellationToken.None);
Assert.Single(results);
Assert.Equal("tenant-example", repository.CapturedTenant);
Assert.Contains("GHSA-XXXX", repository.CapturedAdvisoryKeySearchValues!, StringComparer.Ordinal);
Assert.Contains("ghsa-xxxx", repository.CapturedAdvisoryKeySearchValues!, StringComparer.Ordinal);
Assert.Contains("vendor-x", repository.CapturedAdvisoryKeyVendors!, StringComparer.Ordinal);
}
private static AdvisoryRawService CreateService(RecordingRepository repository)
@@ -86,56 +115,75 @@ public sealed class AdvisoryRawServiceTests
private static AdvisoryRawDocument CreateDocument()
{
using var raw = JsonDocument.Parse("""{"id":"demo"}""");
return new AdvisoryRawDocument(
Tenant: "Tenant-A",
Source: new RawSourceMetadata("Vendor-X", "connector-y", "1.0.0"),
Upstream: new RawUpstreamMetadata(
UpstreamId: "GHSA-xxxx",
DocumentVersion: "1",
RetrievedAt: DateTimeOffset.UtcNow,
ContentHash: "sha256:abc",
Signature: new RawSignatureMetadata(
Present: true,
Format: "dsse",
KeyId: "key-1",
Signature: "base64signature"),
Provenance: ImmutableDictionary<string, string>.Empty),
Content: new RawContent(
Format: "OSV",
SpecVersion: "1.0",
Raw: raw.RootElement.Clone()),
Identifiers: new RawIdentifiers(
Aliases: ImmutableArray.Create("GHSA-xxxx"),
PrimaryId: "GHSA-xxxx"),
Linkset: new RawLinkset
{
Aliases = ImmutableArray<string>.Empty,
PackageUrls = ImmutableArray<string>.Empty,
Cpes = ImmutableArray<string>.Empty,
References = ImmutableArray<RawReference>.Empty,
ReconciledFrom = ImmutableArray<string>.Empty,
Notes = ImmutableDictionary<string, string>.Empty
});
}
return new AdvisoryRawDocument(
Tenant: "Tenant-A",
Source: new RawSourceMetadata("Vendor-X", "connector-y", "1.0.0"),
Upstream: new RawUpstreamMetadata(
UpstreamId: "GHSA-xxxx",
DocumentVersion: "1",
RetrievedAt: DateTimeOffset.UtcNow,
ContentHash: "sha256:abc",
Signature: new RawSignatureMetadata(
Present: true,
Format: "dsse",
KeyId: "key-1",
Signature: "base64signature"),
Provenance: ImmutableDictionary<string, string>.Empty),
Content: new RawContent(
Format: "OSV",
SpecVersion: "1.0",
Raw: raw.RootElement.Clone()),
Identifiers: new RawIdentifiers(
Aliases: ImmutableArray.Create("GHSA-xxxx"),
PrimaryId: "GHSA-xxxx"),
Linkset: new RawLinkset
{
Aliases = ImmutableArray<string>.Empty,
PackageUrls = ImmutableArray<string>.Empty,
Cpes = ImmutableArray<string>.Empty,
References = ImmutableArray<RawReference>.Empty,
ReconciledFrom = ImmutableArray<string>.Empty,
Notes = ImmutableDictionary<string, string>.Empty
},
AdvisoryKey: string.Empty,
Links: ImmutableArray<RawLink>.Empty);
}
private static AdvisoryRawRecord CreateRecord(AdvisoryRawDocument document)
=> new(
Id: "advisory_raw:vendor-x:ghsa-xxxx:sha256-1",
Document: document,
IngestedAt: DateTimeOffset.UtcNow,
CreatedAt: document.Upstream.RetrievedAt);
private static AdvisoryRawRecord CreateRecord(AdvisoryRawDocument document)
{
var canonical = AdvisoryCanonicalizer.Canonicalize(document.Identifiers, document.Source, document.Upstream);
var resolvedDocument = document with
{
AdvisoryKey = string.IsNullOrWhiteSpace(document.AdvisoryKey) ? canonical.AdvisoryKey : document.AdvisoryKey,
Links = document.Links.IsDefaultOrEmpty ? canonical.Links : document.Links
};
return new AdvisoryRawRecord(
Id: "advisory_raw:vendor-x:ghsa-xxxx:sha256-1",
Document: resolvedDocument,
IngestedAt: DateTimeOffset.UtcNow,
CreatedAt: document.Upstream.RetrievedAt);
}
private sealed class RecordingRepository : IAdvisoryRawRepository
{
public AdvisoryRawDocument? CapturedDocument { get; private set; }
public AdvisoryRawUpsertResult? NextResult { get; set; }
public Task<AdvisoryRawUpsertResult> UpsertAsync(AdvisoryRawDocument document, CancellationToken cancellationToken)
{
if (NextResult is null)
{
throw new InvalidOperationException("NextResult must be set before calling UpsertAsync.");
{
public AdvisoryRawDocument? CapturedDocument { get; private set; }
public AdvisoryRawUpsertResult? NextResult { get; set; }
public string? CapturedTenant { get; private set; }
public IReadOnlyCollection<string>? CapturedAdvisoryKeySearchValues { get; private set; }
public IReadOnlyCollection<string>? CapturedAdvisoryKeyVendors { get; private set; }
public IReadOnlyList<AdvisoryRawRecord> AdvisoryKeyResults { get; set; } = Array.Empty<AdvisoryRawRecord>();
public Task<AdvisoryRawUpsertResult> UpsertAsync(AdvisoryRawDocument document, CancellationToken cancellationToken)
{
if (NextResult is null)
{
throw new InvalidOperationException("NextResult must be set before calling UpsertAsync.");
}
CapturedDocument = document;
@@ -145,14 +193,26 @@ public sealed class AdvisoryRawServiceTests
public Task<AdvisoryRawRecord?> FindByIdAsync(string tenant, string id, CancellationToken cancellationToken)
=> throw new NotSupportedException();
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
=> throw new NotSupportedException();
public Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
string tenant,
DateTimeOffset since,
DateTimeOffset until,
IReadOnlyCollection<string> sourceVendors,
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
=> throw new NotSupportedException();
public Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
string tenant,
IReadOnlyCollection<string> searchValues,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken)
{
CapturedTenant = tenant;
CapturedAdvisoryKeySearchValues = searchValues?.ToArray();
CapturedAdvisoryKeyVendors = sourceVendors?.ToArray();
return Task.FromResult(AdvisoryKeyResults);
}
public Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
string tenant,
DateTimeOffset since,
DateTimeOffset until,
IReadOnlyCollection<string> sourceVendors,
CancellationToken cancellationToken)
=> throw new NotSupportedException();
}

View File

@@ -1,5 +1,4 @@
using System;
using System.Linq;
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
@@ -76,29 +75,6 @@ public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime
Assert.True(persisted.BeforeHash.Length > 0);
}
[Fact]
public async Task MergePipeline_IsDeterministicAcrossRuns()
{
await EnsureInitializedAsync();
var merger = _merger!;
var calculator = new CanonicalHashCalculator();
var firstResult = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() });
var secondResult = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() });
var first = firstResult.Advisory;
var second = secondResult.Advisory;
var firstHash = calculator.ComputeHash(first);
var secondHash = calculator.ComputeHash(second);
Assert.Equal(firstHash, secondHash);
Assert.Equal(first.AdvisoryKey, second.AdvisoryKey);
Assert.Equal(first.Aliases.Length, second.Aliases.Length);
Assert.True(first.Aliases.SequenceEqual(second.Aliases));
}
public async Task InitializeAsync()
{
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero))

View File

@@ -69,7 +69,12 @@ public sealed class EnsureAdvisoryObservationsRawLinksetMigrationTests
References = ImmutableArray.Create(new RawReference("advisory", "https://example.test/advisory", "vendor")),
ReconciledFrom = ImmutableArray.Create("connector-y"),
Notes = ImmutableDictionary.CreateRange(new[] { new KeyValuePair<string, string>("range-fixed", "1.0.1") })
});
},
advisoryKey: "CVE-2025-0001",
links: ImmutableArray.Create(
new RawLink("CVE", "CVE-2025-0001"),
new RawLink("GHSA", "GHSA-2025-0001"),
new RawLink("PRIMARY", "CVE-2025-0001")));
await rawRepository.UpsertAsync(rawDocument, CancellationToken.None);
@@ -147,7 +152,11 @@ public sealed class EnsureAdvisoryObservationsRawLinksetMigrationTests
identifiers: new RawIdentifiers(
Aliases: ImmutableArray<string>.Empty,
PrimaryId: "GHSA-9999-0001"),
linkset: new RawLinkset());
linkset: new RawLinkset(),
advisoryKey: "GHSA-9999-0001",
links: ImmutableArray.Create(
new RawLink("GHSA", "GHSA-9999-0001"),
new RawLink("PRIMARY", "GHSA-9999-0001")));
var observationId = "tenant-b:vendor-y:ghsa-9999-0001:sha256-def456";
var document = BuildObservationDocument(

View File

@@ -686,6 +686,21 @@ public sealed class MongoMigrationRunnerTests
{ "notes", new BsonDocument() },
}
},
{ "advisory_key", upstreamId.ToUpperInvariant() },
{
"links",
new BsonArray
{
new BsonDocument
{
{ "scheme", "PRIMARY" },
{ "value", upstreamId.ToUpperInvariant() }
}
}
},
{ "created_at", retrievedAt },
{ "ingested_at", retrievedAt },
{ "supersedes", BsonNull.Value }
};
}
}

View File

@@ -8,11 +8,11 @@ namespace StellaOps.Concelier.WebService.Tests;
public sealed class ConcelierOptionsPostConfigureTests
{
[Fact]
public void Apply_LoadsClientSecretFromRelativeFile()
{
var tempDirectory = Directory.CreateTempSubdirectory();
try
{
public void Apply_LoadsClientSecretFromRelativeFile()
{
var tempDirectory = Directory.CreateTempSubdirectory();
try
{
var secretPath = Path.Combine(tempDirectory.FullName, "authority.secret");
File.WriteAllText(secretPath, " concelier-secret ");
@@ -34,14 +34,22 @@ public sealed class ConcelierOptionsPostConfigureTests
{
Directory.Delete(tempDirectory.FullName, recursive: true);
}
}
}
[Fact]
public void Apply_ThrowsWhenSecretFileMissing()
{
var options = new ConcelierOptions
{
}
}
[Fact]
public void Features_NoMergeEnabled_DefaultsToTrue()
{
var options = new ConcelierOptions();
Assert.True(options.Features.NoMergeEnabled);
}
[Fact]
public void Apply_ThrowsWhenSecretFileMissing()
{
var options = new ConcelierOptions
{
Authority = new ConcelierOptions.AuthorityOptions
{
ClientSecretFile = "missing.secret"

View File

@@ -469,6 +469,55 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
Assert.Empty(firstIds.Intersect(secondIds));
}
[Fact]
public async Task AdvisoryEvidenceEndpoint_ReturnsDocumentsForCanonicalKey()
{
await SeedAdvisoryRawDocumentsAsync(
CreateAdvisoryRawDocument("tenant-a", "vendor-x", "GHSA-2025-0001", "sha256:001", new BsonDocument("id", "GHSA-2025-0001:1")),
CreateAdvisoryRawDocument("tenant-a", "vendor-y", "GHSA-2025-0001", "sha256:002", new BsonDocument("id", "GHSA-2025-0001:2")),
CreateAdvisoryRawDocument("tenant-b", "vendor-x", "GHSA-2025-0001", "sha256:003", new BsonDocument("id", "GHSA-2025-0001:3")));
using var client = _factory.CreateClient();
var response = await client.GetAsync("/vuln/evidence/advisories/ghsa-2025-0001?tenant=tenant-a");
response.EnsureSuccessStatusCode();
var evidence = await response.Content.ReadFromJsonAsync<AdvisoryEvidenceResponse>();
Assert.NotNull(evidence);
Assert.Equal("GHSA-2025-0001", evidence!.AdvisoryKey);
Assert.Equal(2, evidence.Records.Count);
Assert.All(evidence.Records, record => Assert.Equal("tenant-a", record.Tenant));
}
[Fact]
public async Task AdvisoryEvidenceEndpoint_FiltersByVendor()
{
await SeedAdvisoryRawDocumentsAsync(
CreateAdvisoryRawDocument("tenant-a", "vendor-x", "GHSA-2025-0002", "sha256:101", new BsonDocument("id", "GHSA-2025-0002:1")),
CreateAdvisoryRawDocument("tenant-a", "vendor-y", "GHSA-2025-0002", "sha256:102", new BsonDocument("id", "GHSA-2025-0002:2")));
using var client = _factory.CreateClient();
var response = await client.GetAsync("/vuln/evidence/advisories/GHSA-2025-0002?tenant=tenant-a&vendor=vendor-y");
response.EnsureSuccessStatusCode();
var evidence = await response.Content.ReadFromJsonAsync<AdvisoryEvidenceResponse>();
Assert.NotNull(evidence);
var record = Assert.Single(evidence!.Records);
Assert.Equal("vendor-y", record.Document.Source.Vendor);
}
[Fact]
public async Task AdvisoryEvidenceEndpoint_ReturnsNotFoundWhenMissing()
{
await SeedAdvisoryRawDocumentsAsync();
using var client = _factory.CreateClient();
var response = await client.GetAsync("/vuln/evidence/advisories/CVE-2099-9999?tenant=tenant-a");
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
}
[Fact]
public async Task AdvisoryIngestEndpoint_EmitsMetricsWithExpectedTags()
{
@@ -1871,6 +1920,18 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
{ "notes", new BsonDocument() }
}
},
{ "advisory_key", upstreamId.ToUpperInvariant() },
{
"links",
new BsonArray
{
new BsonDocument
{
{ "scheme", "PRIMARY" },
{ "value", upstreamId.ToUpperInvariant() }
}
}
},
{ "supersedes", supersedes is null ? BsonNull.Value : supersedes },
{ "ingested_at", now },
{ "created_at", now }