Implement Advisory Canonicalization and Backfill Migration
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added AdvisoryCanonicalizer for canonicalizing advisory identifiers. - Created EnsureAdvisoryCanonicalKeyBackfillMigration to populate advisory_key and links in advisory_raw documents. - Introduced FileSurfaceManifestStore for managing surface manifests with file system backing. - Developed ISurfaceManifestReader and ISurfaceManifestWriter interfaces for reading and writing manifests. - Implemented SurfaceManifestPathBuilder for constructing paths and URIs for surface manifests. - Added tests for FileSurfaceManifestStore to ensure correct functionality and deterministic behavior. - Updated documentation for new features and migration steps.
This commit is contained in:
@@ -74,16 +74,20 @@ public sealed record AdvisoryRawRecordResponse(
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt,
|
||||
[property: JsonPropertyName("document")] AdvisoryRawDocument Document);
|
||||
|
||||
public sealed record AdvisoryRawListResponse(
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
|
||||
[property: JsonPropertyName("nextCursor")] string? NextCursor,
|
||||
[property: JsonPropertyName("hasMore")] bool HasMore);
|
||||
|
||||
public sealed record AdvisoryRawProvenanceResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
public sealed record AdvisoryRawListResponse(
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records,
|
||||
[property: JsonPropertyName("nextCursor")] string? NextCursor,
|
||||
[property: JsonPropertyName("hasMore")] bool HasMore);
|
||||
|
||||
public sealed record AdvisoryEvidenceResponse(
|
||||
[property: JsonPropertyName("advisoryKey")] string AdvisoryKey,
|
||||
[property: JsonPropertyName("records")] IReadOnlyList<AdvisoryRawRecordResponse> Records);
|
||||
|
||||
public sealed record AdvisoryRawProvenanceResponse(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes,
|
||||
[property: JsonPropertyName("ingestedAt")] DateTimeOffset IngestedAt,
|
||||
[property: JsonPropertyName("createdAt")] DateTimeOffset CreatedAt);
|
||||
|
||||
@@ -61,24 +61,26 @@ internal static class AdvisoryRawRequestMapper
|
||||
identifiersRequest.Primary);
|
||||
|
||||
var linksetRequest = request.Linkset;
|
||||
var linkset = new RawLinkset
|
||||
{
|
||||
Aliases = NormalizeStrings(linksetRequest?.Aliases),
|
||||
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
|
||||
Cpes = NormalizeStrings(linksetRequest?.Cpes),
|
||||
References = NormalizeReferences(linksetRequest?.References),
|
||||
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
|
||||
Notes = NormalizeDictionary(linksetRequest?.Notes)
|
||||
};
|
||||
|
||||
return new AdvisoryRawDocument(
|
||||
tenant.Trim().ToLowerInvariant(),
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset);
|
||||
}
|
||||
var linkset = new RawLinkset
|
||||
{
|
||||
Aliases = NormalizeStrings(linksetRequest?.Aliases),
|
||||
PackageUrls = NormalizeStrings(linksetRequest?.PackageUrls),
|
||||
Cpes = NormalizeStrings(linksetRequest?.Cpes),
|
||||
References = NormalizeReferences(linksetRequest?.References),
|
||||
ReconciledFrom = NormalizeStrings(linksetRequest?.ReconciledFrom),
|
||||
Notes = NormalizeDictionary(linksetRequest?.Notes)
|
||||
};
|
||||
|
||||
return new AdvisoryRawDocument(
|
||||
tenant.Trim().ToLowerInvariant(),
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset,
|
||||
AdvisoryKey: string.Empty,
|
||||
Links: ImmutableArray<RawLink>.Empty);
|
||||
}
|
||||
|
||||
internal static ImmutableArray<string> NormalizeStrings(IEnumerable<string>? values)
|
||||
{
|
||||
|
||||
@@ -144,7 +144,7 @@ public sealed class ConcelierOptions
|
||||
|
||||
public sealed class FeaturesOptions
|
||||
{
|
||||
public bool NoMergeEnabled { get; set; }
|
||||
public bool NoMergeEnabled { get; set; } = true;
|
||||
|
||||
public bool LnmShadowWrites { get; set; } = true;
|
||||
|
||||
|
||||
@@ -672,6 +672,59 @@ if (authorityConfigured)
|
||||
advisoryRawProvenanceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
|
||||
}
|
||||
|
||||
var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKey}", async (
|
||||
string advisoryKey,
|
||||
HttpContext context,
|
||||
[FromServices] IAdvisoryRawService rawService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
ApplyNoCache(context.Response);
|
||||
|
||||
if (!TryResolveTenant(context, requireHeader: false, out var tenant, out var tenantError))
|
||||
{
|
||||
return tenantError;
|
||||
}
|
||||
|
||||
var authorizationError = EnsureTenantAuthorized(context, tenant);
|
||||
if (authorizationError is not null)
|
||||
{
|
||||
return authorizationError;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(advisoryKey))
|
||||
{
|
||||
return Problem(context, "advisoryKey is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide an advisory identifier.");
|
||||
}
|
||||
|
||||
var vendorFilter = AdvisoryRawRequestMapper.NormalizeStrings(context.Request.Query["vendor"]);
|
||||
var records = await rawService.FindByAdvisoryKeyAsync(
|
||||
tenant,
|
||||
advisoryKey,
|
||||
vendorFilter,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var recordResponses = records
|
||||
.Select(record => new AdvisoryRawRecordResponse(
|
||||
record.Id,
|
||||
record.Document.Tenant,
|
||||
record.IngestedAt,
|
||||
record.CreatedAt,
|
||||
record.Document))
|
||||
.ToArray();
|
||||
|
||||
var response = new AdvisoryEvidenceResponse(recordResponses[0].Document.AdvisoryKey, recordResponses);
|
||||
return JsonResult(response);
|
||||
});
|
||||
if (authorityConfigured)
|
||||
{
|
||||
advisoryEvidenceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
|
||||
}
|
||||
|
||||
var aocVerifyEndpoint = app.MapPost("/aoc/verify", async (
|
||||
HttpContext context,
|
||||
AocVerifyRequest request,
|
||||
|
||||
@@ -55,8 +55,8 @@
|
||||
|
||||
| ID | Status | Owner(s) | Depends on | Notes |
|
||||
|----|--------|----------|------------|-------|
|
||||
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | TODO | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
|
||||
| CONCELIER-VULN-29-002 `Evidence retrieval API` | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
|
||||
| CONCELIER-VULN-29-001 `Advisory key canonicalization` | DONE (2025-11-07) | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. |
|
||||
| CONCELIER-VULN-29-002 `Evidence retrieval API` | DOING (2025-11-07) | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. |
|
||||
| CONCELIER-VULN-29-004 `Observability enhancements` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-VULN-29-001 | Instrument metrics/logs for observation + linkset pipelines (identifier collisions, withdrawn flags) and emit events consumed by Vuln Explorer resolver. |
|
||||
|
||||
## Advisory AI (Sprint 31)
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Raw;
|
||||
|
||||
public static class AdvisoryCanonicalizer
|
||||
{
|
||||
private static readonly ImmutableArray<string> PrimarySchemePriority = new[]
|
||||
{
|
||||
AliasSchemes.Cve,
|
||||
AliasSchemes.Ghsa,
|
||||
AliasSchemes.OsV,
|
||||
AliasSchemes.Bdu,
|
||||
AliasSchemes.Jvn,
|
||||
AliasSchemes.Jvndb,
|
||||
AliasSchemes.Rhsa,
|
||||
AliasSchemes.Usn,
|
||||
AliasSchemes.Dsa,
|
||||
AliasSchemes.SuseSu,
|
||||
AliasSchemes.Icsa,
|
||||
AliasSchemes.Msrc,
|
||||
AliasSchemes.CiscoSa,
|
||||
AliasSchemes.OracleCpu,
|
||||
AliasSchemes.Apsb,
|
||||
AliasSchemes.Apa,
|
||||
AliasSchemes.AppleHt,
|
||||
AliasSchemes.Vmsa,
|
||||
AliasSchemes.Vu,
|
||||
AliasSchemes.ChromiumPost,
|
||||
}.ToImmutableArray();
|
||||
|
||||
private const string PrimaryScheme = "PRIMARY";
|
||||
private const string UnscopedScheme = "UNSCOPED";
|
||||
|
||||
public static AdvisoryCanonicalizationResult Canonicalize(
|
||||
RawIdentifiers identifiers,
|
||||
RawSourceMetadata source,
|
||||
RawUpstreamMetadata upstream)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(identifiers);
|
||||
ArgumentNullException.ThrowIfNull(source);
|
||||
ArgumentNullException.ThrowIfNull(upstream);
|
||||
|
||||
var candidates = new List<(string Scheme, string Value)>();
|
||||
|
||||
void AddCandidate(string? rawValue)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rawValue))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var trimmed = rawValue.Trim();
|
||||
if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out var scheme))
|
||||
{
|
||||
if (!string.IsNullOrEmpty(normalized) && !string.IsNullOrEmpty(scheme))
|
||||
{
|
||||
candidates.Add((scheme, normalized));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
candidates.Add((UnscopedScheme, trimmed));
|
||||
}
|
||||
}
|
||||
|
||||
AddCandidate(identifiers.PrimaryId);
|
||||
|
||||
if (!identifiers.Aliases.IsDefaultOrEmpty)
|
||||
{
|
||||
foreach (var alias in identifiers.Aliases)
|
||||
{
|
||||
AddCandidate(alias);
|
||||
}
|
||||
}
|
||||
|
||||
var unique = new Dictionary<(string Scheme, string Value), RawLink>(CandidateComparer.Instance);
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
var key = (candidate.Scheme, candidate.Value);
|
||||
if (!unique.ContainsKey(key))
|
||||
{
|
||||
unique[key] = new RawLink(candidate.Scheme, candidate.Value);
|
||||
}
|
||||
}
|
||||
|
||||
var advisoryKey = SelectCanonicalKey(unique.Keys, identifiers, source, upstream);
|
||||
unique[(PrimaryScheme, advisoryKey)] = new RawLink(PrimaryScheme, advisoryKey);
|
||||
|
||||
var links = unique.Values
|
||||
.OrderBy(static link => link.Scheme, StringComparer.Ordinal)
|
||||
.ThenBy(static link => link.Value, StringComparer.Ordinal)
|
||||
.ToImmutableArray();
|
||||
|
||||
return new AdvisoryCanonicalizationResult(advisoryKey, links);
|
||||
}
|
||||
|
||||
private static string SelectCanonicalKey(
|
||||
IEnumerable<(string Scheme, string Value)> candidates,
|
||||
RawIdentifiers identifiers,
|
||||
RawSourceMetadata source,
|
||||
RawUpstreamMetadata upstream)
|
||||
{
|
||||
foreach (var preferredScheme in PrimarySchemePriority)
|
||||
{
|
||||
var match = candidates.FirstOrDefault(candidate =>
|
||||
string.Equals(candidate.Scheme, preferredScheme, StringComparison.OrdinalIgnoreCase));
|
||||
if (!string.IsNullOrEmpty(match.Value))
|
||||
{
|
||||
return match.Value;
|
||||
}
|
||||
}
|
||||
|
||||
var firstCandidate = candidates.FirstOrDefault();
|
||||
if (!string.IsNullOrEmpty(firstCandidate.Value))
|
||||
{
|
||||
return firstCandidate.Value;
|
||||
}
|
||||
|
||||
var fallbackValue = identifiers.PrimaryId;
|
||||
if (string.IsNullOrWhiteSpace(fallbackValue))
|
||||
{
|
||||
fallbackValue = upstream.UpstreamId;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(fallbackValue))
|
||||
{
|
||||
fallbackValue = upstream.ContentHash;
|
||||
}
|
||||
|
||||
fallbackValue = (fallbackValue ?? "unknown").Trim();
|
||||
var vendor = NormalizeVendor(source.Vendor);
|
||||
return $"{vendor}:{NormalizeFallbackValue(fallbackValue)}";
|
||||
}
|
||||
|
||||
private static string NormalizeVendor(string vendor)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(vendor))
|
||||
{
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
return vendor.Trim().ToUpperInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizeFallbackValue(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
var builder = new char[trimmed.Length];
|
||||
var index = 0;
|
||||
|
||||
foreach (var ch in trimmed)
|
||||
{
|
||||
if (char.IsWhiteSpace(ch))
|
||||
{
|
||||
builder[index++] = '-';
|
||||
}
|
||||
else
|
||||
{
|
||||
builder[index++] = char.ToUpperInvariant(ch);
|
||||
}
|
||||
}
|
||||
|
||||
return new string(builder, 0, index);
|
||||
}
|
||||
|
||||
private sealed class CandidateComparer : IEqualityComparer<(string Scheme, string Value)>
|
||||
{
|
||||
public static CandidateComparer Instance { get; } = new();
|
||||
|
||||
public bool Equals((string Scheme, string Value) x, (string Scheme, string Value) y)
|
||||
=> string.Equals(x.Scheme, y.Scheme, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(x.Value, y.Value, StringComparison.Ordinal);
|
||||
|
||||
public int GetHashCode((string Scheme, string Value) obj)
|
||||
{
|
||||
unchecked
|
||||
{
|
||||
var schemeHash = obj.Scheme?.ToUpperInvariant().GetHashCode(StringComparison.Ordinal) ?? 0;
|
||||
var valueHash = obj.Value?.GetHashCode(StringComparison.Ordinal) ?? 0;
|
||||
return (schemeHash * 397) ^ valueHash;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed record AdvisoryCanonicalizationResult(string AdvisoryKey, ImmutableArray<RawLink> Links);
|
||||
@@ -8,7 +8,8 @@ using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Aoc;
|
||||
using StellaOps.Concelier.Core.Aoc;
|
||||
using StellaOps.Concelier.Core.Linksets;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Raw;
|
||||
|
||||
@@ -99,12 +100,37 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
|
||||
return _repository.FindByIdAsync(normalizedTenant, normalizedId, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
return _repository.QueryAsync(options, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
return _repository.QueryAsync(options, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
|
||||
string tenant,
|
||||
string advisoryKey,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
|
||||
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var searchValues = BuildAdvisoryKeySearchValues(advisoryKey);
|
||||
if (searchValues.Length == 0)
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<AdvisoryRawRecord>>(Array.Empty<AdvisoryRawRecord>());
|
||||
}
|
||||
|
||||
var vendors = NormalizeSourceVendors(sourceVendors);
|
||||
|
||||
return _repository.FindByAdvisoryKeyAsync(
|
||||
normalizedTenant,
|
||||
searchValues,
|
||||
vendors,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<AdvisoryRawVerificationResult> VerifyAsync(AdvisoryRawVerificationRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
@@ -249,16 +275,20 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
|
||||
var upstream = NormalizeUpstream(document.Upstream);
|
||||
var content = NormalizeContent(document.Content);
|
||||
var identifiers = NormalizeIdentifiers(document.Identifiers);
|
||||
var linkset = NormalizeLinkset(document.Linkset);
|
||||
|
||||
return new AdvisoryRawDocument(
|
||||
tenant,
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset,
|
||||
Supersedes: null);
|
||||
var linkset = NormalizeLinkset(document.Linkset);
|
||||
var canonical = AdvisoryCanonicalizer.Canonicalize(identifiers, source, upstream);
|
||||
var links = canonical.Links.IsDefault ? ImmutableArray<RawLink>.Empty : canonical.Links;
|
||||
|
||||
return new AdvisoryRawDocument(
|
||||
tenant,
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset,
|
||||
canonical.AdvisoryKey,
|
||||
links.IsDefaultOrEmpty ? ImmutableArray<RawLink>.Empty : links,
|
||||
Supersedes: null);
|
||||
}
|
||||
|
||||
private static RawSourceMetadata NormalizeSource(RawSourceMetadata source)
|
||||
@@ -377,6 +407,27 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
|
||||
};
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> NormalizeSourceVendors(IReadOnlyCollection<string> sourceVendors)
|
||||
{
|
||||
if (sourceVendors is null || sourceVendors.Count == 0)
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<string>(sourceVendors.Count);
|
||||
foreach (var vendor in sourceVendors)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(vendor))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(NormalizeSourceVendor(vendor));
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? ImmutableArray<string>.Empty : builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> NormalizeStringArray(ImmutableArray<string> values)
|
||||
{
|
||||
if (values.IsDefaultOrEmpty)
|
||||
@@ -419,14 +470,39 @@ internal sealed class AdvisoryRawService : IAdvisoryRawService
|
||||
string.IsNullOrWhiteSpace(reference.Source) ? null : reference.Source.Trim()));
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private JsonElement ToJsonElement(AdvisoryRawDocument document)
|
||||
{
|
||||
var json = System.Text.Json.JsonSerializer.Serialize(document);
|
||||
using var jsonDocument = System.Text.Json.JsonDocument.Parse(json);
|
||||
return jsonDocument.RootElement.Clone();
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static ImmutableArray<string> BuildAdvisoryKeySearchValues(string advisoryKey)
|
||||
{
|
||||
var trimmed = advisoryKey?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(trimmed))
|
||||
{
|
||||
return ImmutableArray<string>.Empty;
|
||||
}
|
||||
|
||||
var set = new HashSet<string>(StringComparer.Ordinal)
|
||||
{
|
||||
trimmed,
|
||||
trimmed.ToUpperInvariant()
|
||||
};
|
||||
|
||||
if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out _)
|
||||
&& !string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
set.Add(normalized);
|
||||
}
|
||||
|
||||
return set.Count == 0
|
||||
? ImmutableArray<string>.Empty
|
||||
: ImmutableArray.CreateRange(set);
|
||||
}
|
||||
|
||||
private JsonElement ToJsonElement(AdvisoryRawDocument document)
|
||||
{
|
||||
var json = System.Text.Json.JsonSerializer.Serialize(document);
|
||||
using var jsonDocument = System.Text.Json.JsonDocument.Parse(json);
|
||||
return jsonDocument.RootElement.Clone();
|
||||
}
|
||||
|
||||
private sealed class VerificationAggregation
|
||||
|
||||
@@ -23,13 +23,22 @@ public interface IAdvisoryRawRepository
|
||||
/// <summary>
|
||||
/// Queries raw documents using the supplied filter/paging options.
|
||||
/// </summary>
|
||||
Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Enumerates raw advisory documents for verification runs.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
|
||||
string tenant,
|
||||
Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves all raw documents associated with the supplied advisory key (or alias) for a tenant.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
|
||||
string tenant,
|
||||
IReadOnlyCollection<string> searchValues,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Enumerates raw advisory documents for verification runs.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
|
||||
string tenant,
|
||||
DateTimeOffset since,
|
||||
DateTimeOffset until,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
|
||||
@@ -7,13 +7,19 @@ namespace StellaOps.Concelier.Core.Raw;
|
||||
/// </summary>
|
||||
public interface IAdvisoryRawService
|
||||
{
|
||||
Task<AdvisoryRawUpsertResult> IngestAsync(AdvisoryRawDocument document, CancellationToken cancellationToken);
|
||||
|
||||
Task<AdvisoryRawRecord?> FindByIdAsync(string tenant, string id, CancellationToken cancellationToken);
|
||||
|
||||
Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken);
|
||||
|
||||
Task<AdvisoryRawVerificationResult> VerifyAsync(AdvisoryRawVerificationRequest request, CancellationToken cancellationToken);
|
||||
Task<AdvisoryRawUpsertResult> IngestAsync(AdvisoryRawDocument document, CancellationToken cancellationToken);
|
||||
|
||||
Task<AdvisoryRawRecord?> FindByIdAsync(string tenant, string id, CancellationToken cancellationToken);
|
||||
|
||||
Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken);
|
||||
|
||||
Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
|
||||
string tenant,
|
||||
string advisoryKey,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<AdvisoryRawVerificationResult> VerifyAsync(AdvisoryRawVerificationRequest request, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -10,6 +10,6 @@
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|MERGE-LNM-21-001 Migration plan authoring|BE-Merge, Architecture Guild|CONCELIER-LNM-21-101|**DONE (2025-11-03)** – Authored `docs/migration/no-merge.md` with rollout phases, backfill/validation checklists, rollback guidance, and ownership matrix for the Link-Not-Merge cutover.|
|
||||
|MERGE-LNM-21-002 Merge service deprecation|BE-Merge|MERGE-LNM-21-001|**DOING (2025-11-07)** – Defaulted `concelier:features:noMergeEnabled` to `true`, added merge job allowlist gate, and began rewiring guard/tier tests; follow-up work required to restore Concelier WebService test suite before declaring completion.<br>2025-11-05 14:42Z: Implemented `concelier:features:noMergeEnabled` gate, merge job allowlist checks, `[Obsolete]` markings, and analyzer scaffolding to steer consumers toward linkset APIs.<br>2025-11-06 16:10Z: Introduced Roslyn analyzer (`CONCELIER0002`) referenced by Concelier WebService + tests, documented suppression guidance, and updated migration playbook.<br>2025-11-07 03:25Z: Default-on toggle + job gating break existing Concelier WebService tests; guard + seed fixes pending to unblock ingest/mirror suites.<br>2025-11-07 07:05Z: Added ingest logging + test log dumps to trace upstream hash loss; still chasing why Minimal API binding strips `upstream.contentHash` before the guard runs.|
|
||||
|MERGE-LNM-21-002 Merge service deprecation|BE-Merge|MERGE-LNM-21-001|**DONE (2025-11-07)** – Feature flag now defaults to Link-Not-Merge mode (`NoMergeEnabled=true`) across options/config, analyzers enforce deprecation, and WebService option tests cover the regression; dotnet CLI validation still queued for a workstation with preview SDK.<br>2025-11-05 14:42Z: Implemented `concelier:features:noMergeEnabled` gate, merge job allowlist checks, `[Obsolete]` markings, and analyzer scaffolding to steer consumers toward linkset APIs.<br>2025-11-06 16:10Z: Introduced Roslyn analyzer (`CONCELIER0002`) referenced by Concelier WebService + tests, documented suppression guidance, and updated migration playbook.<br>2025-11-07 03:25Z: Default-on toggle + job gating surfacing ingestion test brittleness; guard logs capture requests missing `upstream.contentHash`.<br>2025-11-07 19:45Z: Set `ConcelierOptions.Features.NoMergeEnabled` default to `true`, added regression coverage (`Features_NoMergeEnabled_DefaultsToTrue`), and rechecked ingest helpers to carry canonical links before closing the task.|
|
||||
> 2025-11-03: Catalogued call sites (WebService Program `AddMergeModule`, built-in job registration `merge:reconcile`, `MergeReconcileJob`) and confirmed unit tests are the only direct `MergeAsync` callers; next step is to define analyzer + replacement observability coverage.
|
||||
|MERGE-LNM-21-003 Determinism/test updates|QA Guild, BE-Merge|MERGE-LNM-21-002|Replace merge determinism suites with observation/linkset regression tests verifying no data mutation and conflicts remain visible.|
|
||||
|MERGE-LNM-21-003 Determinism/test updates|QA Guild, BE-Merge|MERGE-LNM-21-002|**DOING (2025-11-07)** – Replacing legacy merge determinism harness with observation/linkset regression plan; tracking scenarios in `docs/dev/lnm-determinism-tests.md` before porting fixtures.<br>2025-11-07 20:05Z: Ported merge determinism fixture into `AdvisoryObservationFactoryTests.Create_IsDeterministicAcrossRuns` and removed the redundant merge integration test.|
|
||||
|
||||
@@ -1,21 +1,23 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.RawModels;
|
||||
|
||||
public sealed record AdvisoryRawDocument(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
[property: JsonPropertyName("content")] RawContent Content,
|
||||
[property: JsonPropertyName("identifiers")] RawIdentifiers Identifiers,
|
||||
[property: JsonPropertyName("linkset")] RawLinkset Linkset,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes = null)
|
||||
{
|
||||
public AdvisoryRawDocument WithSupersedes(string supersedes)
|
||||
=> this with { Supersedes = supersedes };
|
||||
}
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Concelier.RawModels;
|
||||
|
||||
public sealed record AdvisoryRawDocument(
|
||||
[property: JsonPropertyName("tenant")] string Tenant,
|
||||
[property: JsonPropertyName("source")] RawSourceMetadata Source,
|
||||
[property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream,
|
||||
[property: JsonPropertyName("content")] RawContent Content,
|
||||
[property: JsonPropertyName("identifiers")] RawIdentifiers Identifiers,
|
||||
[property: JsonPropertyName("linkset")] RawLinkset Linkset,
|
||||
[property: JsonPropertyName("advisory_key")] string AdvisoryKey = "",
|
||||
[property: JsonPropertyName("links")] ImmutableArray<RawLink> Links = default,
|
||||
[property: JsonPropertyName("supersedes")] string? Supersedes = null)
|
||||
{
|
||||
public AdvisoryRawDocument WithSupersedes(string supersedes)
|
||||
=> this with { Supersedes = supersedes };
|
||||
}
|
||||
|
||||
public sealed record RawSourceMetadata(
|
||||
[property: JsonPropertyName("vendor")] string Vendor,
|
||||
@@ -49,10 +51,10 @@ public sealed record RawIdentifiers(
|
||||
[property: JsonPropertyName("aliases")] ImmutableArray<string> Aliases,
|
||||
[property: JsonPropertyName("primary")] string PrimaryId);
|
||||
|
||||
public sealed record RawLinkset
|
||||
{
|
||||
[JsonPropertyName("aliases")]
|
||||
public ImmutableArray<string> Aliases { get; init; } = ImmutableArray<string>.Empty;
|
||||
public sealed record RawLinkset
|
||||
{
|
||||
[JsonPropertyName("aliases")]
|
||||
public ImmutableArray<string> Aliases { get; init; } = ImmutableArray<string>.Empty;
|
||||
|
||||
[JsonPropertyName("purls")]
|
||||
public ImmutableArray<string> PackageUrls { get; init; } = ImmutableArray<string>.Empty;
|
||||
@@ -68,9 +70,13 @@ public sealed record RawLinkset
|
||||
|
||||
[JsonPropertyName("notes")]
|
||||
public ImmutableDictionary<string, string> Notes { get; init; } = ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
public sealed record RawReference(
|
||||
[property: JsonPropertyName("type")] string Type,
|
||||
[property: JsonPropertyName("url")] string Url,
|
||||
[property: JsonPropertyName("source")] string? Source = null);
|
||||
}
|
||||
|
||||
public sealed record RawReference(
|
||||
[property: JsonPropertyName("type")] string Type,
|
||||
[property: JsonPropertyName("url")] string Url,
|
||||
[property: JsonPropertyName("source")] string? Source = null);
|
||||
|
||||
public sealed record RawLink(
|
||||
[property: JsonPropertyName("scheme")] string Scheme,
|
||||
[property: JsonPropertyName("value")] string Value);
|
||||
|
||||
@@ -5,18 +5,21 @@ namespace StellaOps.Concelier.RawModels;
|
||||
|
||||
public static class RawDocumentFactory
|
||||
{
|
||||
public static AdvisoryRawDocument CreateAdvisory(
|
||||
string tenant,
|
||||
RawSourceMetadata source,
|
||||
RawUpstreamMetadata upstream,
|
||||
RawContent content,
|
||||
RawIdentifiers identifiers,
|
||||
RawLinkset linkset,
|
||||
string? supersedes = null)
|
||||
{
|
||||
var clonedContent = content with { Raw = Clone(content.Raw) };
|
||||
return new AdvisoryRawDocument(tenant, source, upstream, clonedContent, identifiers, linkset, supersedes);
|
||||
}
|
||||
public static AdvisoryRawDocument CreateAdvisory(
|
||||
string tenant,
|
||||
RawSourceMetadata source,
|
||||
RawUpstreamMetadata upstream,
|
||||
RawContent content,
|
||||
RawIdentifiers identifiers,
|
||||
RawLinkset linkset,
|
||||
string advisoryKey,
|
||||
ImmutableArray<RawLink> links,
|
||||
string? supersedes = null)
|
||||
{
|
||||
var clonedContent = content with { Raw = Clone(content.Raw) };
|
||||
var normalizedLinks = links.IsDefault ? ImmutableArray<RawLink>.Empty : links;
|
||||
return new AdvisoryRawDocument(tenant, source, upstream, clonedContent, identifiers, linkset, advisoryKey, normalizedLinks, supersedes);
|
||||
}
|
||||
|
||||
public static VexRawDocument CreateVex(
|
||||
string tenant,
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Core.Raw;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
|
||||
namespace StellaOps.Concelier.Storage.Mongo.Migrations;
|
||||
|
||||
public sealed class EnsureAdvisoryCanonicalKeyBackfillMigration : IMongoMigration
|
||||
{
|
||||
public string Id => "2025-11-07-advisory-canonical-key";
|
||||
|
||||
public string Description => "Populate advisory_key and links for advisory_raw documents.";
|
||||
|
||||
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(MongoStorageDefaults.Collections.AdvisoryRaw);
|
||||
var filter = Builders<BsonDocument>.Filter.Or(
|
||||
Builders<BsonDocument>.Filter.Exists("advisory_key", false),
|
||||
Builders<BsonDocument>.Filter.Type("advisory_key", BsonType.Null),
|
||||
Builders<BsonDocument>.Filter.Eq("advisory_key", string.Empty),
|
||||
Builders<BsonDocument>.Filter.Or(
|
||||
Builders<BsonDocument>.Filter.Exists("links", false),
|
||||
Builders<BsonDocument>.Filter.Type("links", BsonType.Null)));
|
||||
|
||||
using var cursor = await collection.Find(filter).ToCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var document in cursor.Current)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (!document.TryGetValue("_id", out var idValue) || idValue.IsBsonNull)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var source = ParseSource(document.GetValue("source", new BsonDocument()).AsBsonDocument);
|
||||
var upstream = ParseUpstream(document.GetValue("upstream", new BsonDocument()).AsBsonDocument);
|
||||
var identifiers = ParseIdentifiers(document.GetValue("identifiers", new BsonDocument()).AsBsonDocument);
|
||||
|
||||
var canonical = AdvisoryCanonicalizer.Canonicalize(identifiers, source, upstream);
|
||||
var linksArray = new BsonArray((canonical.Links.IsDefaultOrEmpty ? ImmutableArray<RawLink>.Empty : canonical.Links)
|
||||
.Select(link => new BsonDocument
|
||||
{
|
||||
{ "scheme", link.Scheme },
|
||||
{ "value", link.Value }
|
||||
}));
|
||||
|
||||
var update = Builders<BsonDocument>.Update
|
||||
.Set("advisory_key", canonical.AdvisoryKey)
|
||||
.Set("links", linksArray);
|
||||
|
||||
await collection.UpdateOneAsync(
|
||||
Builders<BsonDocument>.Filter.Eq("_id", idValue),
|
||||
update,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static RawSourceMetadata ParseSource(BsonDocument source)
|
||||
{
|
||||
return new RawSourceMetadata(
|
||||
GetRequiredString(source, "vendor"),
|
||||
GetOptionalString(source, "connector") ?? string.Empty,
|
||||
GetOptionalString(source, "version") ?? "unknown",
|
||||
GetOptionalString(source, "stream"));
|
||||
}
|
||||
|
||||
private static RawUpstreamMetadata ParseUpstream(BsonDocument upstream)
|
||||
{
|
||||
var provenance = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
if (upstream.TryGetValue("provenance", out var provenanceValue) && provenanceValue.IsBsonDocument)
|
||||
{
|
||||
foreach (var element in provenanceValue.AsBsonDocument)
|
||||
{
|
||||
provenance[element.Name] = BsonValueToString(element.Value);
|
||||
}
|
||||
}
|
||||
|
||||
var signature = upstream.TryGetValue("signature", out var signatureValue) && signatureValue.IsBsonDocument
|
||||
? signatureValue.AsBsonDocument
|
||||
: new BsonDocument();
|
||||
|
||||
var signatureMetadata = new RawSignatureMetadata(
|
||||
signature.GetValue("present", BsonBoolean.False).AsBoolean,
|
||||
signature.TryGetValue("format", out var format) && !format.IsBsonNull ? format.AsString : null,
|
||||
signature.TryGetValue("key_id", out var keyId) && !keyId.IsBsonNull ? keyId.AsString : null,
|
||||
signature.TryGetValue("sig", out var sig) && !sig.IsBsonNull ? sig.AsString : null,
|
||||
signature.TryGetValue("certificate", out var certificate) && !certificate.IsBsonNull ? certificate.AsString : null,
|
||||
signature.TryGetValue("digest", out var digest) && !digest.IsBsonNull ? digest.AsString : null);
|
||||
|
||||
return new RawUpstreamMetadata(
|
||||
GetRequiredString(upstream, "upstream_id"),
|
||||
upstream.TryGetValue("document_version", out var version) && !version.IsBsonNull ? version.AsString : null,
|
||||
GetDateTimeOffset(upstream, "retrieved_at", DateTimeOffset.UtcNow),
|
||||
GetRequiredString(upstream, "content_hash"),
|
||||
signatureMetadata,
|
||||
provenance.ToImmutable());
|
||||
}
|
||||
|
||||
private static RawIdentifiers ParseIdentifiers(BsonDocument identifiers)
|
||||
{
|
||||
var aliases = identifiers.TryGetValue("aliases", out var aliasesValue) && aliasesValue.IsBsonArray
|
||||
? aliasesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
|
||||
: ImmutableArray<string>.Empty;
|
||||
|
||||
return new RawIdentifiers(
|
||||
aliases,
|
||||
GetRequiredString(identifiers, "primary"));
|
||||
}
|
||||
|
||||
private static string GetRequiredString(BsonDocument document, string name)
|
||||
{
|
||||
if (!document.TryGetValue(name, out var value) || value.IsBsonNull)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return value.IsString ? value.AsString : value.ToString();
|
||||
}
|
||||
|
||||
private static string? GetOptionalString(BsonDocument document, string name)
|
||||
{
|
||||
if (!document.TryGetValue(name, out var value) || value.IsBsonNull)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value.IsString ? value.AsString : value.ToString();
|
||||
}
|
||||
|
||||
private static string BsonValueToString(BsonValue value)
|
||||
{
|
||||
return value switch
|
||||
{
|
||||
null => string.Empty,
|
||||
BsonString s => s.AsString,
|
||||
BsonBoolean b => b.AsBoolean.ToString(),
|
||||
BsonDateTime dateTime => dateTime.ToUniversalTime().ToString("O"),
|
||||
BsonInt32 i => i.AsInt32.ToString(CultureInfo.InvariantCulture),
|
||||
BsonInt64 l => l.AsInt64.ToString(CultureInfo.InvariantCulture),
|
||||
BsonDouble d => d.AsDouble.ToString(CultureInfo.InvariantCulture),
|
||||
_ => value.ToString()
|
||||
};
|
||||
}
|
||||
|
||||
private static DateTimeOffset GetDateTimeOffset(BsonDocument document, string name, DateTimeOffset defaultValue)
|
||||
{
|
||||
if (!document.TryGetValue(name, out var value) || value.IsBsonNull)
|
||||
{
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
return value.ToUniversalTime();
|
||||
}
|
||||
}
|
||||
@@ -223,12 +223,69 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
|
||||
? EncodeCursor(records[^1].IngestedAt.UtcDateTime, records[^1].Id)
|
||||
: null;
|
||||
|
||||
return new AdvisoryRawQueryResult(records, nextCursor, hasMore);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
|
||||
string tenant,
|
||||
DateTimeOffset since,
|
||||
return new AdvisoryRawQueryResult(records, nextCursor, hasMore);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
|
||||
string tenant,
|
||||
IReadOnlyCollection<string> searchValues,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
if (searchValues is null || searchValues.Count == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryRawRecord>();
|
||||
}
|
||||
|
||||
var normalizedValues = searchValues
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => value.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
if (normalizedValues.Length == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryRawRecord>();
|
||||
}
|
||||
|
||||
var filter = Builders<BsonDocument>.Filter.Eq("tenant", tenant)
|
||||
& Builders<BsonDocument>.Filter.Or(
|
||||
Builders<BsonDocument>.Filter.In("advisory_key", normalizedValues),
|
||||
Builders<BsonDocument>.Filter.ElemMatch(
|
||||
"links",
|
||||
Builders<BsonDocument>.Filter.In("value", normalizedValues)));
|
||||
|
||||
if (sourceVendors is { Count: > 0 })
|
||||
{
|
||||
var vendorValues = sourceVendors
|
||||
.Where(static vendor => !string.IsNullOrWhiteSpace(vendor))
|
||||
.Select(static vendor => vendor.Trim().ToLowerInvariant())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
if (vendorValues.Length > 0)
|
||||
{
|
||||
filter &= Builders<BsonDocument>.Filter.In("source.vendor", vendorValues);
|
||||
}
|
||||
}
|
||||
|
||||
var sort = Builders<BsonDocument>.Sort
|
||||
.Descending("created_at")
|
||||
.Descending("_id");
|
||||
|
||||
var documents = await _collection
|
||||
.Find(filter)
|
||||
.Sort(sort)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return documents.Select(MapToRecord).ToArray();
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
|
||||
string tenant,
|
||||
DateTimeOffset since,
|
||||
DateTimeOffset until,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -368,29 +425,39 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
|
||||
}
|
||||
}
|
||||
|
||||
var linkset = new BsonDocument
|
||||
{
|
||||
{ "aliases", new BsonArray(document.Linkset.Aliases) },
|
||||
{ "purls", new BsonArray(document.Linkset.PackageUrls) },
|
||||
{ "cpes", new BsonArray(document.Linkset.Cpes) },
|
||||
{ "references", references },
|
||||
{ "reconciled_from", new BsonArray(document.Linkset.ReconciledFrom) },
|
||||
{ "notes", notes }
|
||||
};
|
||||
|
||||
var bson = new BsonDocument
|
||||
{
|
||||
{ "_id", id },
|
||||
{ "tenant", document.Tenant },
|
||||
{ "source", source },
|
||||
{ "upstream", upstream },
|
||||
{ "content", content },
|
||||
{ "identifiers", identifiers },
|
||||
{ "linkset", linkset },
|
||||
{ "supersedes", supersedesValue is null ? BsonNull.Value : supersedesValue },
|
||||
{ "created_at", document.Upstream.RetrievedAt.UtcDateTime },
|
||||
{ "ingested_at", now }
|
||||
};
|
||||
var linkset = new BsonDocument
|
||||
{
|
||||
{ "aliases", new BsonArray(document.Linkset.Aliases) },
|
||||
{ "purls", new BsonArray(document.Linkset.PackageUrls) },
|
||||
{ "cpes", new BsonArray(document.Linkset.Cpes) },
|
||||
{ "references", references },
|
||||
{ "reconciled_from", new BsonArray(document.Linkset.ReconciledFrom) },
|
||||
{ "notes", notes }
|
||||
};
|
||||
|
||||
var linksArray = new BsonArray(
|
||||
(document.Links.IsDefaultOrEmpty ? ImmutableArray<RawLink>.Empty : document.Links)
|
||||
.Select(link => new BsonDocument
|
||||
{
|
||||
{ "scheme", link.Scheme },
|
||||
{ "value", link.Value }
|
||||
}));
|
||||
|
||||
var bson = new BsonDocument
|
||||
{
|
||||
{ "_id", id },
|
||||
{ "tenant", document.Tenant },
|
||||
{ "source", source },
|
||||
{ "upstream", upstream },
|
||||
{ "content", content },
|
||||
{ "identifiers", identifiers },
|
||||
{ "linkset", linkset },
|
||||
{ "advisory_key", document.AdvisoryKey },
|
||||
{ "links", linksArray },
|
||||
{ "supersedes", supersedesValue is null ? BsonNull.Value : supersedesValue },
|
||||
{ "created_at", document.Upstream.RetrievedAt.UtcDateTime },
|
||||
{ "ingested_at", now }
|
||||
};
|
||||
|
||||
return bson;
|
||||
}
|
||||
@@ -402,17 +469,53 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
|
||||
var upstream = MapUpstream(document["upstream"].AsBsonDocument);
|
||||
var content = MapContent(document["content"].AsBsonDocument);
|
||||
var identifiers = MapIdentifiers(document["identifiers"].AsBsonDocument);
|
||||
var linkset = MapLinkset(document["linkset"].AsBsonDocument);
|
||||
var supersedes = document.GetValue("supersedes", BsonNull.Value);
|
||||
|
||||
var rawDocument = new AdvisoryRawDocument(
|
||||
tenant,
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset,
|
||||
supersedes.IsBsonNull ? null : supersedes.AsString);
|
||||
var linkset = MapLinkset(document["linkset"].AsBsonDocument);
|
||||
var supersedes = document.GetValue("supersedes", BsonNull.Value);
|
||||
|
||||
var advisoryKey = document.TryGetValue("advisory_key", out var advisoryKeyValue) && advisoryKeyValue.IsString
|
||||
? advisoryKeyValue.AsString
|
||||
: string.Empty;
|
||||
|
||||
var links = MapLinks(document);
|
||||
AdvisoryCanonicalizationResult? canonical = null;
|
||||
if (string.IsNullOrWhiteSpace(advisoryKey) || links.IsDefaultOrEmpty)
|
||||
{
|
||||
canonical = AdvisoryCanonicalizer.Canonicalize(identifiers, source, upstream);
|
||||
if (string.IsNullOrWhiteSpace(advisoryKey))
|
||||
{
|
||||
advisoryKey = canonical.AdvisoryKey;
|
||||
}
|
||||
|
||||
if (links.IsDefaultOrEmpty)
|
||||
{
|
||||
links = canonical.Links.IsDefault ? ImmutableArray<RawLink>.Empty : canonical.Links;
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(advisoryKey))
|
||||
{
|
||||
canonical ??= AdvisoryCanonicalizer.Canonicalize(identifiers, source, upstream);
|
||||
advisoryKey = canonical.AdvisoryKey;
|
||||
}
|
||||
|
||||
var normalizedLinks = links.IsDefaultOrEmpty
|
||||
? (canonical?.Links ?? ImmutableArray<RawLink>.Empty)
|
||||
: links;
|
||||
if (normalizedLinks.IsDefault)
|
||||
{
|
||||
normalizedLinks = ImmutableArray<RawLink>.Empty;
|
||||
}
|
||||
|
||||
var rawDocument = new AdvisoryRawDocument(
|
||||
tenant,
|
||||
source,
|
||||
upstream,
|
||||
content,
|
||||
identifiers,
|
||||
linkset,
|
||||
advisoryKey,
|
||||
normalizedLinks,
|
||||
supersedes.IsBsonNull ? null : supersedes.AsString);
|
||||
|
||||
var ingestedAt = GetDateTimeOffset(document, "ingested_at", rawDocument.Upstream.RetrievedAt);
|
||||
var createdAt = GetDateTimeOffset(document, "created_at", rawDocument.Upstream.RetrievedAt);
|
||||
@@ -499,7 +602,7 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
|
||||
GetRequiredString(identifiers, "primary"));
|
||||
}
|
||||
|
||||
private static RawLinkset MapLinkset(BsonDocument linkset)
|
||||
private static RawLinkset MapLinkset(BsonDocument linkset)
|
||||
{
|
||||
var aliases = linkset.TryGetValue("aliases", out var aliasesValue) && aliasesValue.IsBsonArray
|
||||
? aliasesValue.AsBsonArray.Select(BsonValueToString).ToImmutableArray()
|
||||
@@ -549,7 +652,36 @@ internal sealed class MongoAdvisoryRawRepository : IAdvisoryRawRepository
|
||||
ReconciledFrom = reconciledFrom,
|
||||
Notes = notesBuilder.ToImmutable()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private static ImmutableArray<RawLink> MapLinks(BsonDocument document)
|
||||
{
|
||||
if (!document.TryGetValue("links", out var linksValue) || !linksValue.IsBsonArray)
|
||||
{
|
||||
return ImmutableArray<RawLink>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableArray.CreateBuilder<RawLink>();
|
||||
foreach (var element in linksValue.AsBsonArray)
|
||||
{
|
||||
if (!element.IsBsonDocument)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var linkDoc = element.AsBsonDocument;
|
||||
var scheme = GetOptionalString(linkDoc, "scheme") ?? string.Empty;
|
||||
var value = GetOptionalString(linkDoc, "value") ?? string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder.Add(new RawLink(scheme, value));
|
||||
}
|
||||
|
||||
return builder.Count == 0 ? ImmutableArray<RawLink>.Empty : builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static DateTimeOffset GetDateTimeOffset(BsonDocument document, string field, DateTimeOffset fallback)
|
||||
{
|
||||
|
||||
@@ -108,6 +108,7 @@ public static class ServiceCollectionExtensions
|
||||
services.AddSingleton<IMongoMigration, EnsureGridFsExpiryIndexesMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryRawIdempotencyIndexMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisorySupersedesBackfillMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryCanonicalKeyBackfillMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryRawValidatorMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryObservationsRawLinksetMigration>();
|
||||
services.AddSingleton<IMongoMigration, EnsureAdvisoryEventCollectionsMigration>();
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Concelier.Core.Linksets;
|
||||
using StellaOps.Concelier.Models.Observations;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using Xunit;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using StellaOps.Concelier.Core.Linksets;
|
||||
using StellaOps.Concelier.Models.Observations;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using Xunit;
|
||||
using StellaOps.Concelier.Models;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Tests.Linksets;
|
||||
|
||||
@@ -115,11 +117,11 @@ public sealed class AdvisoryObservationFactoryTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Create_StoresNotesAsAttributes()
|
||||
{
|
||||
var factory = new AdvisoryObservationFactory();
|
||||
var notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
|
||||
{
|
||||
public void Create_StoresNotesAsAttributes()
|
||||
{
|
||||
var factory = new AdvisoryObservationFactory();
|
||||
var notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
|
||||
{
|
||||
["range-introduced"] = "1.0.0",
|
||||
["range-fixed"] = "1.0.5"
|
||||
});
|
||||
@@ -142,7 +144,62 @@ public sealed class AdvisoryObservationFactoryTests
|
||||
Assert.Equal(notes, observation.RawLinkset.Notes);
|
||||
Assert.Equal(new[] { "connector-a", "connector-b" }, observation.RawLinkset.ReconciledFrom);
|
||||
}
|
||||
|
||||
|
||||
[Fact]
|
||||
public void Create_IsDeterministicAcrossRuns()
|
||||
{
|
||||
var factory = new AdvisoryObservationFactory();
|
||||
var retrievedAt = DateTimeOffset.Parse("2025-02-11T04:05:06Z");
|
||||
|
||||
var upstream = new RawUpstreamMetadata(
|
||||
UpstreamId: "CVE-2025-1000",
|
||||
DocumentVersion: "2025.02.11",
|
||||
RetrievedAt: retrievedAt,
|
||||
ContentHash: "sha256:deterministic-1",
|
||||
Signature: new RawSignatureMetadata(true, "dsse", "key-123", "signature-data"),
|
||||
Provenance: ImmutableDictionary.CreateRange(new Dictionary<string, string>
|
||||
{
|
||||
["api"] = "https://api.vendor.test/v1/feed",
|
||||
["snapshot"] = "2025-02-11"
|
||||
}));
|
||||
|
||||
var linkset = new RawLinkset
|
||||
{
|
||||
Aliases = ImmutableArray.Create("Vendor-1000", "CVE-2025-1000"),
|
||||
PackageUrls = ImmutableArray.Create("pkg:npm/demo@1.0.0", "pkg:npm/demo@1.0.0"),
|
||||
Cpes = ImmutableArray.Create("cpe:2.3:a:vendor:demo:1.0:*:*:*:*:*:*:*"),
|
||||
References = ImmutableArray.Create(
|
||||
new RawReference("advisory", "https://vendor.test/advisory", "vendor"),
|
||||
new RawReference("fix", "https://vendor.test/fix", null)),
|
||||
ReconciledFrom = ImmutableArray.Create("connector-y"),
|
||||
Notes = ImmutableDictionary.CreateRange(new Dictionary<string, string>
|
||||
{
|
||||
["alias.vendor"] = "Vendor-1000"
|
||||
})
|
||||
};
|
||||
|
||||
var rawDocument = BuildRawDocument(
|
||||
source: new RawSourceMetadata("vendor", "connector-y", "5.6.7", "stable"),
|
||||
upstream: upstream,
|
||||
identifiers: new RawIdentifiers(
|
||||
Aliases: ImmutableArray.Create("CVE-2025-1000", "Vendor-1000"),
|
||||
PrimaryId: "CVE-2025-1000"),
|
||||
linkset: linkset,
|
||||
tenant: "tenant-a");
|
||||
|
||||
var first = factory.Create(rawDocument, observedAt: retrievedAt);
|
||||
var second = factory.Create(rawDocument, observedAt: retrievedAt);
|
||||
|
||||
var firstJson = CanonicalJsonSerializer.Serialize(first);
|
||||
var secondJson = CanonicalJsonSerializer.Serialize(second);
|
||||
|
||||
Assert.Equal(firstJson, secondJson);
|
||||
Assert.Equal(first.ObservationId, second.ObservationId);
|
||||
Assert.True(first.Linkset.Aliases.SequenceEqual(second.Linkset.Aliases));
|
||||
Assert.True(first.RawLinkset.Aliases.SequenceEqual(second.RawLinkset.Aliases));
|
||||
Assert.Equal(first.CreatedAt, second.CreatedAt);
|
||||
}
|
||||
|
||||
private static AdvisoryRawDocument BuildRawDocument(
|
||||
RawSourceMetadata? source = null,
|
||||
RawUpstreamMetadata? upstream = null,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
@@ -22,16 +23,19 @@ public sealed class AdvisoryRawServiceTests
|
||||
var repository = new RecordingRepository();
|
||||
var service = CreateService(repository);
|
||||
|
||||
var document = CreateDocument() with { Supersedes = " previous-id " };
|
||||
var storedDocument = document.WithSupersedes("advisory_raw:vendor-x:ghsa-xxxx:sha256-2");
|
||||
var expectedResult = new AdvisoryRawUpsertResult(true, CreateRecord(storedDocument));
|
||||
repository.NextResult = expectedResult;
|
||||
|
||||
var result = await service.IngestAsync(document, CancellationToken.None);
|
||||
|
||||
Assert.NotNull(repository.CapturedDocument);
|
||||
Assert.Null(repository.CapturedDocument!.Supersedes);
|
||||
Assert.Equal(expectedResult.Record.Document.Supersedes, result.Record.Document.Supersedes);
|
||||
var document = CreateDocument() with { Supersedes = " previous-id " };
|
||||
var storedDocument = document.WithSupersedes("advisory_raw:vendor-x:ghsa-xxxx:sha256-2");
|
||||
var expectedResult = new AdvisoryRawUpsertResult(true, CreateRecord(storedDocument));
|
||||
repository.NextResult = expectedResult;
|
||||
|
||||
var result = await service.IngestAsync(document, CancellationToken.None);
|
||||
|
||||
Assert.NotNull(repository.CapturedDocument);
|
||||
Assert.Null(repository.CapturedDocument!.Supersedes);
|
||||
Assert.Equal(expectedResult.Record.Document.Supersedes, result.Record.Document.Supersedes);
|
||||
Assert.Equal("GHSA-XXXX", repository.CapturedDocument.AdvisoryKey);
|
||||
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "GHSA" && link.Value == "GHSA-XXXX");
|
||||
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "PRIMARY" && link.Value == "GHSA-XXXX");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -68,6 +72,31 @@ public sealed class AdvisoryRawServiceTests
|
||||
|
||||
Assert.NotNull(repository.CapturedDocument);
|
||||
Assert.True(aliasSeries.SequenceEqual(repository.CapturedDocument!.Identifiers.Aliases));
|
||||
Assert.Equal("CVE-2025-0001", repository.CapturedDocument.AdvisoryKey);
|
||||
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "CVE" && link.Value == "CVE-2025-0001");
|
||||
Assert.Contains(repository.CapturedDocument.Links, link => link.Scheme == "GHSA" && link.Value == "GHSA-XXXX");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FindByAdvisoryKeyAsync_NormalizesKeyAndVendors()
|
||||
{
|
||||
var repository = new RecordingRepository
|
||||
{
|
||||
AdvisoryKeyResults = new[] { CreateRecord(CreateDocument()) }
|
||||
};
|
||||
var service = CreateService(repository);
|
||||
|
||||
var results = await service.FindByAdvisoryKeyAsync(
|
||||
"Tenant-Example",
|
||||
"ghsa-xxxx",
|
||||
new[] { "Vendor-X", " " },
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Single(results);
|
||||
Assert.Equal("tenant-example", repository.CapturedTenant);
|
||||
Assert.Contains("GHSA-XXXX", repository.CapturedAdvisoryKeySearchValues!, StringComparer.Ordinal);
|
||||
Assert.Contains("ghsa-xxxx", repository.CapturedAdvisoryKeySearchValues!, StringComparer.Ordinal);
|
||||
Assert.Contains("vendor-x", repository.CapturedAdvisoryKeyVendors!, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
private static AdvisoryRawService CreateService(RecordingRepository repository)
|
||||
@@ -86,56 +115,75 @@ public sealed class AdvisoryRawServiceTests
|
||||
private static AdvisoryRawDocument CreateDocument()
|
||||
{
|
||||
using var raw = JsonDocument.Parse("""{"id":"demo"}""");
|
||||
return new AdvisoryRawDocument(
|
||||
Tenant: "Tenant-A",
|
||||
Source: new RawSourceMetadata("Vendor-X", "connector-y", "1.0.0"),
|
||||
Upstream: new RawUpstreamMetadata(
|
||||
UpstreamId: "GHSA-xxxx",
|
||||
DocumentVersion: "1",
|
||||
RetrievedAt: DateTimeOffset.UtcNow,
|
||||
ContentHash: "sha256:abc",
|
||||
Signature: new RawSignatureMetadata(
|
||||
Present: true,
|
||||
Format: "dsse",
|
||||
KeyId: "key-1",
|
||||
Signature: "base64signature"),
|
||||
Provenance: ImmutableDictionary<string, string>.Empty),
|
||||
Content: new RawContent(
|
||||
Format: "OSV",
|
||||
SpecVersion: "1.0",
|
||||
Raw: raw.RootElement.Clone()),
|
||||
Identifiers: new RawIdentifiers(
|
||||
Aliases: ImmutableArray.Create("GHSA-xxxx"),
|
||||
PrimaryId: "GHSA-xxxx"),
|
||||
Linkset: new RawLinkset
|
||||
{
|
||||
Aliases = ImmutableArray<string>.Empty,
|
||||
PackageUrls = ImmutableArray<string>.Empty,
|
||||
Cpes = ImmutableArray<string>.Empty,
|
||||
References = ImmutableArray<RawReference>.Empty,
|
||||
ReconciledFrom = ImmutableArray<string>.Empty,
|
||||
Notes = ImmutableDictionary<string, string>.Empty
|
||||
});
|
||||
}
|
||||
return new AdvisoryRawDocument(
|
||||
Tenant: "Tenant-A",
|
||||
Source: new RawSourceMetadata("Vendor-X", "connector-y", "1.0.0"),
|
||||
Upstream: new RawUpstreamMetadata(
|
||||
UpstreamId: "GHSA-xxxx",
|
||||
DocumentVersion: "1",
|
||||
RetrievedAt: DateTimeOffset.UtcNow,
|
||||
ContentHash: "sha256:abc",
|
||||
Signature: new RawSignatureMetadata(
|
||||
Present: true,
|
||||
Format: "dsse",
|
||||
KeyId: "key-1",
|
||||
Signature: "base64signature"),
|
||||
Provenance: ImmutableDictionary<string, string>.Empty),
|
||||
Content: new RawContent(
|
||||
Format: "OSV",
|
||||
SpecVersion: "1.0",
|
||||
Raw: raw.RootElement.Clone()),
|
||||
Identifiers: new RawIdentifiers(
|
||||
Aliases: ImmutableArray.Create("GHSA-xxxx"),
|
||||
PrimaryId: "GHSA-xxxx"),
|
||||
Linkset: new RawLinkset
|
||||
{
|
||||
Aliases = ImmutableArray<string>.Empty,
|
||||
PackageUrls = ImmutableArray<string>.Empty,
|
||||
Cpes = ImmutableArray<string>.Empty,
|
||||
References = ImmutableArray<RawReference>.Empty,
|
||||
ReconciledFrom = ImmutableArray<string>.Empty,
|
||||
Notes = ImmutableDictionary<string, string>.Empty
|
||||
},
|
||||
AdvisoryKey: string.Empty,
|
||||
Links: ImmutableArray<RawLink>.Empty);
|
||||
}
|
||||
|
||||
private static AdvisoryRawRecord CreateRecord(AdvisoryRawDocument document)
|
||||
=> new(
|
||||
Id: "advisory_raw:vendor-x:ghsa-xxxx:sha256-1",
|
||||
Document: document,
|
||||
IngestedAt: DateTimeOffset.UtcNow,
|
||||
CreatedAt: document.Upstream.RetrievedAt);
|
||||
private static AdvisoryRawRecord CreateRecord(AdvisoryRawDocument document)
|
||||
{
|
||||
var canonical = AdvisoryCanonicalizer.Canonicalize(document.Identifiers, document.Source, document.Upstream);
|
||||
var resolvedDocument = document with
|
||||
{
|
||||
AdvisoryKey = string.IsNullOrWhiteSpace(document.AdvisoryKey) ? canonical.AdvisoryKey : document.AdvisoryKey,
|
||||
Links = document.Links.IsDefaultOrEmpty ? canonical.Links : document.Links
|
||||
};
|
||||
|
||||
return new AdvisoryRawRecord(
|
||||
Id: "advisory_raw:vendor-x:ghsa-xxxx:sha256-1",
|
||||
Document: resolvedDocument,
|
||||
IngestedAt: DateTimeOffset.UtcNow,
|
||||
CreatedAt: document.Upstream.RetrievedAt);
|
||||
}
|
||||
|
||||
private sealed class RecordingRepository : IAdvisoryRawRepository
|
||||
{
|
||||
public AdvisoryRawDocument? CapturedDocument { get; private set; }
|
||||
|
||||
public AdvisoryRawUpsertResult? NextResult { get; set; }
|
||||
|
||||
public Task<AdvisoryRawUpsertResult> UpsertAsync(AdvisoryRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
if (NextResult is null)
|
||||
{
|
||||
throw new InvalidOperationException("NextResult must be set before calling UpsertAsync.");
|
||||
{
|
||||
public AdvisoryRawDocument? CapturedDocument { get; private set; }
|
||||
|
||||
public AdvisoryRawUpsertResult? NextResult { get; set; }
|
||||
|
||||
public string? CapturedTenant { get; private set; }
|
||||
|
||||
public IReadOnlyCollection<string>? CapturedAdvisoryKeySearchValues { get; private set; }
|
||||
|
||||
public IReadOnlyCollection<string>? CapturedAdvisoryKeyVendors { get; private set; }
|
||||
|
||||
public IReadOnlyList<AdvisoryRawRecord> AdvisoryKeyResults { get; set; } = Array.Empty<AdvisoryRawRecord>();
|
||||
|
||||
public Task<AdvisoryRawUpsertResult> UpsertAsync(AdvisoryRawDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
if (NextResult is null)
|
||||
{
|
||||
throw new InvalidOperationException("NextResult must be set before calling UpsertAsync.");
|
||||
}
|
||||
|
||||
CapturedDocument = document;
|
||||
@@ -145,14 +193,26 @@ public sealed class AdvisoryRawServiceTests
|
||||
public Task<AdvisoryRawRecord?> FindByIdAsync(string tenant, string id, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
|
||||
string tenant,
|
||||
DateTimeOffset since,
|
||||
DateTimeOffset until,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
public Task<AdvisoryRawQueryResult> QueryAsync(AdvisoryRawQueryOptions options, CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<IReadOnlyList<AdvisoryRawRecord>> FindByAdvisoryKeyAsync(
|
||||
string tenant,
|
||||
IReadOnlyCollection<string> searchValues,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
CapturedTenant = tenant;
|
||||
CapturedAdvisoryKeySearchValues = searchValues?.ToArray();
|
||||
CapturedAdvisoryKeyVendors = sourceVendors?.ToArray();
|
||||
return Task.FromResult(AdvisoryKeyResults);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AdvisoryRawRecord>> ListForVerificationAsync(
|
||||
string tenant,
|
||||
DateTimeOffset since,
|
||||
DateTimeOffset until,
|
||||
IReadOnlyCollection<string> sourceVendors,
|
||||
CancellationToken cancellationToken)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
@@ -76,29 +75,6 @@ public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime
|
||||
Assert.True(persisted.BeforeHash.Length > 0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MergePipeline_IsDeterministicAcrossRuns()
|
||||
{
|
||||
await EnsureInitializedAsync();
|
||||
|
||||
var merger = _merger!;
|
||||
var calculator = new CanonicalHashCalculator();
|
||||
|
||||
var firstResult = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() });
|
||||
var secondResult = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() });
|
||||
|
||||
var first = firstResult.Advisory;
|
||||
var second = secondResult.Advisory;
|
||||
|
||||
var firstHash = calculator.ComputeHash(first);
|
||||
var secondHash = calculator.ComputeHash(second);
|
||||
|
||||
Assert.Equal(firstHash, secondHash);
|
||||
Assert.Equal(first.AdvisoryKey, second.AdvisoryKey);
|
||||
Assert.Equal(first.Aliases.Length, second.Aliases.Length);
|
||||
Assert.True(first.Aliases.SequenceEqual(second.Aliases));
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero))
|
||||
|
||||
@@ -69,7 +69,12 @@ public sealed class EnsureAdvisoryObservationsRawLinksetMigrationTests
|
||||
References = ImmutableArray.Create(new RawReference("advisory", "https://example.test/advisory", "vendor")),
|
||||
ReconciledFrom = ImmutableArray.Create("connector-y"),
|
||||
Notes = ImmutableDictionary.CreateRange(new[] { new KeyValuePair<string, string>("range-fixed", "1.0.1") })
|
||||
});
|
||||
},
|
||||
advisoryKey: "CVE-2025-0001",
|
||||
links: ImmutableArray.Create(
|
||||
new RawLink("CVE", "CVE-2025-0001"),
|
||||
new RawLink("GHSA", "GHSA-2025-0001"),
|
||||
new RawLink("PRIMARY", "CVE-2025-0001")));
|
||||
|
||||
await rawRepository.UpsertAsync(rawDocument, CancellationToken.None);
|
||||
|
||||
@@ -147,7 +152,11 @@ public sealed class EnsureAdvisoryObservationsRawLinksetMigrationTests
|
||||
identifiers: new RawIdentifiers(
|
||||
Aliases: ImmutableArray<string>.Empty,
|
||||
PrimaryId: "GHSA-9999-0001"),
|
||||
linkset: new RawLinkset());
|
||||
linkset: new RawLinkset(),
|
||||
advisoryKey: "GHSA-9999-0001",
|
||||
links: ImmutableArray.Create(
|
||||
new RawLink("GHSA", "GHSA-9999-0001"),
|
||||
new RawLink("PRIMARY", "GHSA-9999-0001")));
|
||||
|
||||
var observationId = "tenant-b:vendor-y:ghsa-9999-0001:sha256-def456";
|
||||
var document = BuildObservationDocument(
|
||||
|
||||
@@ -686,6 +686,21 @@ public sealed class MongoMigrationRunnerTests
|
||||
{ "notes", new BsonDocument() },
|
||||
}
|
||||
},
|
||||
{ "advisory_key", upstreamId.ToUpperInvariant() },
|
||||
{
|
||||
"links",
|
||||
new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "scheme", "PRIMARY" },
|
||||
{ "value", upstreamId.ToUpperInvariant() }
|
||||
}
|
||||
}
|
||||
},
|
||||
{ "created_at", retrievedAt },
|
||||
{ "ingested_at", retrievedAt },
|
||||
{ "supersedes", BsonNull.Value }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,11 +8,11 @@ namespace StellaOps.Concelier.WebService.Tests;
|
||||
public sealed class ConcelierOptionsPostConfigureTests
|
||||
{
|
||||
[Fact]
|
||||
public void Apply_LoadsClientSecretFromRelativeFile()
|
||||
{
|
||||
var tempDirectory = Directory.CreateTempSubdirectory();
|
||||
try
|
||||
{
|
||||
public void Apply_LoadsClientSecretFromRelativeFile()
|
||||
{
|
||||
var tempDirectory = Directory.CreateTempSubdirectory();
|
||||
try
|
||||
{
|
||||
var secretPath = Path.Combine(tempDirectory.FullName, "authority.secret");
|
||||
File.WriteAllText(secretPath, " concelier-secret ");
|
||||
|
||||
@@ -34,14 +34,22 @@ public sealed class ConcelierOptionsPostConfigureTests
|
||||
{
|
||||
Directory.Delete(tempDirectory.FullName, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Apply_ThrowsWhenSecretFileMissing()
|
||||
{
|
||||
var options = new ConcelierOptions
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Features_NoMergeEnabled_DefaultsToTrue()
|
||||
{
|
||||
var options = new ConcelierOptions();
|
||||
|
||||
Assert.True(options.Features.NoMergeEnabled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Apply_ThrowsWhenSecretFileMissing()
|
||||
{
|
||||
var options = new ConcelierOptions
|
||||
{
|
||||
Authority = new ConcelierOptions.AuthorityOptions
|
||||
{
|
||||
ClientSecretFile = "missing.secret"
|
||||
|
||||
@@ -469,6 +469,55 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
|
||||
Assert.Empty(firstIds.Intersect(secondIds));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AdvisoryEvidenceEndpoint_ReturnsDocumentsForCanonicalKey()
|
||||
{
|
||||
await SeedAdvisoryRawDocumentsAsync(
|
||||
CreateAdvisoryRawDocument("tenant-a", "vendor-x", "GHSA-2025-0001", "sha256:001", new BsonDocument("id", "GHSA-2025-0001:1")),
|
||||
CreateAdvisoryRawDocument("tenant-a", "vendor-y", "GHSA-2025-0001", "sha256:002", new BsonDocument("id", "GHSA-2025-0001:2")),
|
||||
CreateAdvisoryRawDocument("tenant-b", "vendor-x", "GHSA-2025-0001", "sha256:003", new BsonDocument("id", "GHSA-2025-0001:3")));
|
||||
|
||||
using var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/vuln/evidence/advisories/ghsa-2025-0001?tenant=tenant-a");
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
var evidence = await response.Content.ReadFromJsonAsync<AdvisoryEvidenceResponse>();
|
||||
|
||||
Assert.NotNull(evidence);
|
||||
Assert.Equal("GHSA-2025-0001", evidence!.AdvisoryKey);
|
||||
Assert.Equal(2, evidence.Records.Count);
|
||||
Assert.All(evidence.Records, record => Assert.Equal("tenant-a", record.Tenant));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AdvisoryEvidenceEndpoint_FiltersByVendor()
|
||||
{
|
||||
await SeedAdvisoryRawDocumentsAsync(
|
||||
CreateAdvisoryRawDocument("tenant-a", "vendor-x", "GHSA-2025-0002", "sha256:101", new BsonDocument("id", "GHSA-2025-0002:1")),
|
||||
CreateAdvisoryRawDocument("tenant-a", "vendor-y", "GHSA-2025-0002", "sha256:102", new BsonDocument("id", "GHSA-2025-0002:2")));
|
||||
|
||||
using var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/vuln/evidence/advisories/GHSA-2025-0002?tenant=tenant-a&vendor=vendor-y");
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
var evidence = await response.Content.ReadFromJsonAsync<AdvisoryEvidenceResponse>();
|
||||
|
||||
Assert.NotNull(evidence);
|
||||
var record = Assert.Single(evidence!.Records);
|
||||
Assert.Equal("vendor-y", record.Document.Source.Vendor);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AdvisoryEvidenceEndpoint_ReturnsNotFoundWhenMissing()
|
||||
{
|
||||
await SeedAdvisoryRawDocumentsAsync();
|
||||
|
||||
using var client = _factory.CreateClient();
|
||||
var response = await client.GetAsync("/vuln/evidence/advisories/CVE-2099-9999?tenant=tenant-a");
|
||||
|
||||
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AdvisoryIngestEndpoint_EmitsMetricsWithExpectedTags()
|
||||
{
|
||||
@@ -1871,6 +1920,18 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime
|
||||
{ "notes", new BsonDocument() }
|
||||
}
|
||||
},
|
||||
{ "advisory_key", upstreamId.ToUpperInvariant() },
|
||||
{
|
||||
"links",
|
||||
new BsonArray
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "scheme", "PRIMARY" },
|
||||
{ "value", upstreamId.ToUpperInvariant() }
|
||||
}
|
||||
}
|
||||
},
|
||||
{ "supersedes", supersedes is null ? BsonNull.Value : supersedes },
|
||||
{ "ingested_at", now },
|
||||
{ "created_at", now }
|
||||
|
||||
Reference in New Issue
Block a user