interim commit
Some checks failed
Build Test Deploy / build-test (push) Has been cancelled
Build Test Deploy / docs (push) Has been cancelled
Build Test Deploy / deploy (push) Has been cancelled

This commit is contained in:
2025-10-08 03:17:52 +03:00
parent 6cbfd47ecd
commit cd0e64f391
21 changed files with 998 additions and 51 deletions

View File

@@ -71,6 +71,17 @@ public sealed class JsonFeedExporter : IFeedExporter
return;
}
var resetBaseline = existingState is null
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
if (existingState is not null
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
{
resetBaseline = true;
}
await _stateManager.StoreFullExportAsync(
ExporterId,
exportId,
@@ -78,6 +89,7 @@ public sealed class JsonFeedExporter : IFeedExporter
cursor: digest,
targetRepository: _options.TargetRepository,
exporterVersion: _exporterVersion,
resetBaseline: resetBaseline,
cancellationToken: cancellationToken).ConfigureAwait(false);
await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false);

View File

@@ -16,6 +16,7 @@ public sealed class TrivyDbExportPlannerTests
Assert.Equal("sha256:abcd", plan.TreeDigest);
Assert.Null(plan.BaseExportId);
Assert.Null(plan.BaseManifestDigest);
Assert.True(plan.ResetBaseline);
}
[Fact]
@@ -39,6 +40,7 @@ public sealed class TrivyDbExportPlannerTests
Assert.Equal("sha256:unchanged", plan.TreeDigest);
Assert.Equal("20240810T000000Z", plan.BaseExportId);
Assert.Equal("sha256:base", plan.BaseManifestDigest);
Assert.False(plan.ResetBaseline);
}
[Fact]
@@ -62,5 +64,12 @@ public sealed class TrivyDbExportPlannerTests
Assert.Equal("sha256:new", plan.TreeDigest);
Assert.Equal("20240810T000000Z", plan.BaseExportId);
Assert.Equal("sha256:base", plan.BaseManifestDigest);
Assert.False(plan.ResetBaseline);
var deltaState = state with { LastDeltaDigest = "sha256:delta" };
var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer");
Assert.Equal(TrivyDbExportMode.Full, deltaPlan.Mode);
Assert.True(deltaPlan.ResetBaseline);
}
}

View File

@@ -281,6 +281,77 @@ public sealed class TrivyDbFeedExporterTests : IDisposable
Assert.Empty(orasPusher.Pushes);
}
[Fact]
public async Task ExportAsync_ResetsBaselineWhenDeltaChainExists()
{
var advisory = CreateSampleAdvisory("CVE-2024-5000", "Baseline reset");
var advisoryStore = new StubAdvisoryStore(advisory);
var optionsValue = new TrivyDbExportOptions
{
OutputRoot = _root,
ReferencePrefix = "example/trivy",
Json = new JsonExportOptions
{
OutputRoot = _jsonRoot,
MaintainLatestSymlink = false,
},
KeepWorkingTree = false,
TargetRepository = "registry.example/trivy",
};
var options = Options.Create(optionsValue);
var packageBuilder = new TrivyDbPackageBuilder();
var ociWriter = new TrivyDbOciWriter();
var planner = new TrivyDbExportPlanner();
var stateStore = new InMemoryExportStateStore();
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-22T00:00:00Z", CultureInfo.InvariantCulture));
var existingRecord = new ExportStateRecord(
TrivyDbFeedExporter.ExporterId,
BaseExportId: "20240919T120000Z",
BaseDigest: "sha256:base",
LastFullDigest: "sha256:base",
LastDeltaDigest: "sha256:delta",
ExportCursor: "sha256:old",
TargetRepository: "registry.example/trivy",
ExporterVersion: "0.9.0",
UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30));
await stateStore.UpsertAsync(existingRecord, CancellationToken.None);
var stateManager = new ExportStateManager(stateStore, timeProvider);
var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new
{
Version = 2,
NextUpdate = "2024-09-23T00:00:00Z",
UpdatedAt = "2024-09-22T00:00:00Z",
});
var builder = new StubTrivyDbBuilder(_root, builderMetadata);
var orasPusher = new StubTrivyDbOrasPusher();
var exporter = new TrivyDbFeedExporter(
advisoryStore,
new VulnListJsonExportPathResolver(),
options,
packageBuilder,
ociWriter,
stateManager,
planner,
builder,
orasPusher,
NullLogger<TrivyDbFeedExporter>.Instance,
timeProvider);
using var provider = new ServiceCollection().BuildServiceProvider();
await exporter.ExportAsync(provider, CancellationToken.None);
var updated = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None);
Assert.NotNull(updated);
Assert.Equal("20240922T000000Z", updated!.BaseExportId);
Assert.Equal(updated.BaseDigest, updated.LastFullDigest);
Assert.Null(updated.LastDeltaDigest);
Assert.NotEqual("sha256:old", updated.ExportCursor);
Assert.Equal("registry.example/trivy", updated.TargetRepository);
}
private static Advisory CreateSampleAdvisory(
string advisoryKey = "CVE-2024-9999",
string title = "Trivy Export Test")

View File

@@ -8,6 +8,6 @@
|Offline bundle toggle|BE-Export|Exporters|DONE Deterministic OCI layout bundle emitted when enabled.|
|Deterministic ordering of advisories|BE-Export|Models|DONE exporter now loads advisories, sorts by advisoryKey, and emits sorted JSON trees with deterministic OCI payloads.|
|End-to-end tests with small dataset|QA|Exporters|DONE added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.|
|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DOING `ExportStateManager` keeps stable base export metadata; delta reset remains pending.|
|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.|
|Streamed package building to avoid large copies|BE-Export|Exporters|DONE metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.|
|Plan incremental/delta exports|BE-Export|Exporters|TODO design reuse of existing blobs/layers when inputs unchanged instead of rewriting full trees each run.|

View File

@@ -4,4 +4,5 @@ public sealed record TrivyDbExportPlan(
TrivyDbExportMode Mode,
string TreeDigest,
string? BaseExportId,
string? BaseManifestDigest);
string? BaseManifestDigest,
bool ResetBaseline);

View File

@@ -11,7 +11,12 @@ public sealed class TrivyDbExportPlanner
if (existingState is null)
{
return new TrivyDbExportPlan(TrivyDbExportMode.Full, treeDigest, BaseExportId: null, BaseManifestDigest: null);
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
BaseExportId: null,
BaseManifestDigest: null,
ResetBaseline: true);
}
if (string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal))
@@ -20,14 +25,18 @@ public sealed class TrivyDbExportPlanner
TrivyDbExportMode.Skip,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest);
existingState.LastFullDigest,
ResetBaseline: false);
}
var resetBaseline = existingState.LastDeltaDigest is not null;
// Placeholder for future delta support current behavior always rebuilds when tree changes.
return new TrivyDbExportPlan(
TrivyDbExportMode.Full,
treeDigest,
existingState.BaseExportId,
existingState.LastFullDigest);
existingState.LastFullDigest,
resetBaseline);
}
}

View File

@@ -130,6 +130,18 @@ public sealed class TrivyDbFeedExporter : IFeedExporter
exportId,
ociResult.ManifestDigest);
var resetBaseline = plan.ResetBaseline
|| existingState is null
|| string.IsNullOrWhiteSpace(existingState.BaseExportId)
|| string.IsNullOrWhiteSpace(existingState.BaseDigest);
if (existingState is not null
&& !string.IsNullOrWhiteSpace(_options.TargetRepository)
&& !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal))
{
resetBaseline = true;
}
await _stateManager.StoreFullExportAsync(
ExporterId,
exportId,
@@ -137,6 +149,7 @@ public sealed class TrivyDbFeedExporter : IFeedExporter
cursor: treeDigest,
targetRepository: _options.TargetRepository,
exporterVersion: _exporterVersion,
resetBaseline: resetBaseline,
cancellationToken: cancellationToken).ConfigureAwait(false);
await CreateOfflineBundleAsync(destination, exportId, exportedAt, cancellationToken).ConfigureAwait(false);

View File

@@ -0,0 +1,91 @@
using System;
using System.Linq;
using StellaOps.Feedser.Merge.Identity;
using StellaOps.Feedser.Models;
using Xunit;
namespace StellaOps.Feedser.Merge.Tests;
public sealed class AdvisoryIdentityResolverTests
{
private readonly AdvisoryIdentityResolver _resolver = new();
[Fact]
public void Resolve_GroupsBySharedCveAlias()
{
var nvd = CreateAdvisory("CVE-2025-1234", aliases: new[] { "CVE-2025-1234" }, source: "nvd");
var vendor = CreateAdvisory("VSA-2025-01", aliases: new[] { "CVE-2025-1234", "VSA-2025-01" }, source: "vendor");
var clusters = _resolver.Resolve(new[] { nvd, vendor });
var cluster = Assert.Single(clusters);
Assert.Equal("CVE-2025-1234", cluster.AdvisoryKey);
Assert.Equal(2, cluster.Advisories.Length);
Assert.True(cluster.Aliases.Any(alias => alias.Value == "CVE-2025-1234"));
}
[Fact]
public void Resolve_PrefersPsirtAliasWhenNoCve()
{
var vendor = CreateAdvisory("VMSA-2025-0001", aliases: new[] { "VMSA-2025-0001" }, source: "vmware");
var osv = CreateAdvisory("OSV-2025-1", aliases: new[] { "OSV-2025-1", "GHSA-xxxx-yyyy-zzzz", "VMSA-2025-0001" }, source: "osv");
var clusters = _resolver.Resolve(new[] { vendor, osv });
var cluster = Assert.Single(clusters);
Assert.Equal("VMSA-2025-0001", cluster.AdvisoryKey);
Assert.Equal(2, cluster.Advisories.Length);
Assert.True(cluster.Aliases.Any(alias => alias.Value == "VMSA-2025-0001"));
}
[Fact]
public void Resolve_FallsBackToGhsaWhenOnlyGhsaPresent()
{
var ghsa = CreateAdvisory("GHSA-aaaa-bbbb-cccc", aliases: new[] { "GHSA-aaaa-bbbb-cccc" }, source: "ghsa");
var osv = CreateAdvisory("OSV-2025-99", aliases: new[] { "OSV-2025-99", "GHSA-aaaa-bbbb-cccc" }, source: "osv");
var clusters = _resolver.Resolve(new[] { ghsa, osv });
var cluster = Assert.Single(clusters);
Assert.Equal("GHSA-aaaa-bbbb-cccc", cluster.AdvisoryKey);
Assert.Equal(2, cluster.Advisories.Length);
Assert.True(cluster.Aliases.Any(alias => alias.Value == "GHSA-aaaa-bbbb-cccc"));
}
[Fact]
public void Resolve_GroupsByKeyWhenNoAliases()
{
var first = CreateAdvisory("custom-1", aliases: Array.Empty<string>(), source: "source-a");
var second = CreateAdvisory("custom-1", aliases: Array.Empty<string>(), source: "source-b");
var clusters = _resolver.Resolve(new[] { first, second });
var cluster = Assert.Single(clusters);
Assert.Equal("custom-1", cluster.AdvisoryKey);
Assert.Equal(2, cluster.Advisories.Length);
Assert.Contains(cluster.Aliases, alias => alias.Value == "custom-1");
}
private static Advisory CreateAdvisory(string key, string[] aliases, string source)
{
var provenance = new[]
{
new AdvisoryProvenance(source, "mapping", key, DateTimeOffset.UtcNow),
};
return new Advisory(
key,
$"{key} title",
$"{key} summary",
"en",
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow,
null,
exploitKnown: false,
aliases,
Array.Empty<AdvisoryReference>(),
Array.Empty<AffectedPackage>(),
Array.Empty<CvssMetric>(),
provenance);
}
}

View File

@@ -0,0 +1,56 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Feedser.Models;
namespace StellaOps.Feedser.Merge.Identity;
/// <summary>
/// Represents a connected component of advisories that refer to the same vulnerability.
/// </summary>
public sealed class AdvisoryIdentityCluster
{
public AdvisoryIdentityCluster(string advisoryKey, IEnumerable<Advisory> advisories, IEnumerable<AliasIdentity> aliases)
{
AdvisoryKey = !string.IsNullOrWhiteSpace(advisoryKey)
? advisoryKey.Trim()
: throw new ArgumentException("Canonical advisory key must be provided.", nameof(advisoryKey));
var advisoriesArray = (advisories ?? throw new ArgumentNullException(nameof(advisories)))
.Where(static advisory => advisory is not null)
.OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase)
.ThenBy(static advisory => advisory.Provenance.Length)
.ThenBy(static advisory => advisory.Title, StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
if (advisoriesArray.IsDefaultOrEmpty)
{
throw new ArgumentException("At least one advisory is required for a cluster.", nameof(advisories));
}
var aliasArray = (aliases ?? throw new ArgumentNullException(nameof(aliases)))
.Where(static alias => alias is not null && !string.IsNullOrWhiteSpace(alias.Value))
.GroupBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase)
.Select(static group =>
{
var representative = group
.OrderBy(static entry => entry.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase)
.ThenBy(static entry => entry.Value, StringComparer.OrdinalIgnoreCase)
.First();
return representative;
})
.OrderBy(static alias => alias.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase)
.ThenBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
Advisories = advisoriesArray;
Aliases = aliasArray;
}
public string AdvisoryKey { get; }
public ImmutableArray<Advisory> Advisories { get; }
public ImmutableArray<AliasIdentity> Aliases { get; }
}

View File

@@ -0,0 +1,303 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Runtime.CompilerServices;
using StellaOps.Feedser.Models;
namespace StellaOps.Feedser.Merge.Identity;
/// <summary>
/// Builds an alias-driven identity graph that groups advisories referring to the same vulnerability.
/// </summary>
public sealed class AdvisoryIdentityResolver
{
private static readonly string[] CanonicalAliasPriority =
{
AliasSchemes.Cve,
AliasSchemes.Rhsa,
AliasSchemes.Usn,
AliasSchemes.Dsa,
AliasSchemes.SuseSu,
AliasSchemes.Msrc,
AliasSchemes.CiscoSa,
AliasSchemes.OracleCpu,
AliasSchemes.Vmsa,
AliasSchemes.Apsb,
AliasSchemes.Apa,
AliasSchemes.AppleHt,
AliasSchemes.ChromiumPost,
AliasSchemes.Icsa,
AliasSchemes.Jvndb,
AliasSchemes.Jvn,
AliasSchemes.Bdu,
AliasSchemes.Vu,
AliasSchemes.Ghsa,
AliasSchemes.OsV,
};
/// <summary>
/// Groups the provided advisories into identity clusters using normalized aliases.
/// </summary>
public IReadOnlyList<AdvisoryIdentityCluster> Resolve(IEnumerable<Advisory> advisories)
{
ArgumentNullException.ThrowIfNull(advisories);
var materialized = advisories
.Where(static advisory => advisory is not null)
.Distinct()
.ToArray();
if (materialized.Length == 0)
{
return Array.Empty<AdvisoryIdentityCluster>();
}
var aliasIndex = BuildAliasIndex(materialized);
var visited = new HashSet<Advisory>();
var clusters = new List<AdvisoryIdentityCluster>();
foreach (var advisory in materialized)
{
if (!visited.Add(advisory))
{
continue;
}
var component = TraverseComponent(advisory, visited, aliasIndex);
var key = DetermineCanonicalKey(component);
var aliases = component
.SelectMany(static entry => entry.Aliases)
.Select(static alias => new AliasIdentity(alias.Normalized, alias.Scheme));
clusters.Add(new AdvisoryIdentityCluster(key, component.Select(static entry => entry.Advisory), aliases));
}
return clusters
.OrderBy(static cluster => cluster.AdvisoryKey, StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
}
private static Dictionary<string, List<AdvisoryAliasEntry>> BuildAliasIndex(IEnumerable<Advisory> advisories)
{
var index = new Dictionary<string, List<AdvisoryAliasEntry>>(StringComparer.OrdinalIgnoreCase);
foreach (var advisory in advisories)
{
foreach (var alias in ExtractAliases(advisory))
{
if (!index.TryGetValue(alias.Normalized, out var list))
{
list = new List<AdvisoryAliasEntry>();
index[alias.Normalized] = list;
}
list.Add(new AdvisoryAliasEntry(advisory, alias.Normalized, alias.Scheme));
}
}
return index;
}
private static IReadOnlyList<AliasBinding> TraverseComponent(
Advisory root,
HashSet<Advisory> visited,
Dictionary<string, List<AdvisoryAliasEntry>> aliasIndex)
{
var stack = new Stack<Advisory>();
stack.Push(root);
var bindings = new Dictionary<Advisory, AliasBinding>(ReferenceEqualityComparer<Advisory>.Instance);
while (stack.Count > 0)
{
var advisory = stack.Pop();
if (!bindings.TryGetValue(advisory, out var binding))
{
binding = new AliasBinding(advisory);
bindings[advisory] = binding;
}
foreach (var alias in ExtractAliases(advisory))
{
binding.AddAlias(alias.Normalized, alias.Scheme);
if (!aliasIndex.TryGetValue(alias.Normalized, out var neighbors))
{
continue;
}
foreach (var neighbor in neighbors.Select(static entry => entry.Advisory))
{
if (visited.Add(neighbor))
{
stack.Push(neighbor);
}
if (!bindings.TryGetValue(neighbor, out var neighborBinding))
{
neighborBinding = new AliasBinding(neighbor);
bindings[neighbor] = neighborBinding;
}
neighborBinding.AddAlias(alias.Normalized, alias.Scheme);
}
}
}
return bindings.Values
.OrderBy(static binding => binding.Advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase)
.ToImmutableArray();
}
private static string DetermineCanonicalKey(IReadOnlyList<AliasBinding> component)
{
var aliases = component
.SelectMany(static binding => binding.Aliases)
.Where(static alias => !string.IsNullOrWhiteSpace(alias.Normalized))
.ToArray();
foreach (var scheme in CanonicalAliasPriority)
{
var candidate = aliases
.Where(alias => string.Equals(alias.Scheme, scheme, StringComparison.OrdinalIgnoreCase))
.Select(static alias => alias.Normalized)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.FirstOrDefault();
if (candidate is not null)
{
return candidate;
}
}
var fallbackAlias = aliases
.Select(static alias => alias.Normalized)
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
.FirstOrDefault();
if (!string.IsNullOrWhiteSpace(fallbackAlias))
{
return fallbackAlias;
}
var advisoryKey = component
.Select(static binding => binding.Advisory.AdvisoryKey)
.Where(static value => !string.IsNullOrWhiteSpace(value))
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
.FirstOrDefault();
if (!string.IsNullOrWhiteSpace(advisoryKey))
{
return advisoryKey.Trim();
}
throw new InvalidOperationException("Unable to determine canonical advisory key for cluster.");
}
private static IEnumerable<AliasProjection> ExtractAliases(Advisory advisory)
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var candidate in EnumerateAliasCandidates(advisory))
{
if (string.IsNullOrWhiteSpace(candidate))
{
continue;
}
var trimmed = candidate.Trim();
if (!seen.Add(trimmed))
{
continue;
}
if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out var scheme) &&
!string.IsNullOrWhiteSpace(normalized))
{
yield return new AliasProjection(normalized.Trim(), string.IsNullOrWhiteSpace(scheme) ? null : scheme);
}
else if (!string.IsNullOrWhiteSpace(normalized))
{
yield return new AliasProjection(normalized.Trim(), null);
}
}
}
private static IEnumerable<string> EnumerateAliasCandidates(Advisory advisory)
{
if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey))
{
yield return advisory.AdvisoryKey;
}
if (!advisory.Aliases.IsDefaultOrEmpty)
{
foreach (var alias in advisory.Aliases)
{
if (!string.IsNullOrWhiteSpace(alias))
{
yield return alias;
}
}
}
}
private readonly record struct AdvisoryAliasEntry(Advisory Advisory, string Normalized, string? Scheme);
private readonly record struct AliasProjection(string Normalized, string? Scheme);
private sealed class AliasBinding
{
private readonly HashSet<AliasProjection> _aliases = new(HashSetAliasComparer.Instance);
public AliasBinding(Advisory advisory)
{
Advisory = advisory ?? throw new ArgumentNullException(nameof(advisory));
}
public Advisory Advisory { get; }
public IReadOnlyCollection<AliasProjection> Aliases => _aliases;
public void AddAlias(string normalized, string? scheme)
{
if (string.IsNullOrWhiteSpace(normalized))
{
return;
}
_aliases.Add(new AliasProjection(normalized.Trim(), scheme is null ? null : scheme.Trim()));
}
}
private sealed class HashSetAliasComparer : IEqualityComparer<AliasProjection>
{
public static readonly HashSetAliasComparer Instance = new();
public bool Equals(AliasProjection x, AliasProjection y)
=> string.Equals(x.Normalized, y.Normalized, StringComparison.OrdinalIgnoreCase)
&& string.Equals(x.Scheme, y.Scheme, StringComparison.OrdinalIgnoreCase);
public int GetHashCode(AliasProjection obj)
{
var hash = StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Normalized);
if (!string.IsNullOrWhiteSpace(obj.Scheme))
{
hash = HashCode.Combine(hash, StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Scheme));
}
return hash;
}
}
private sealed class ReferenceEqualityComparer<T> : IEqualityComparer<T>
where T : class
{
public static readonly ReferenceEqualityComparer<T> Instance = new();
public bool Equals(T? x, T? y) => ReferenceEquals(x, y);
public int GetHashCode(T obj) => RuntimeHelpers.GetHashCode(obj);
}
}

View File

@@ -0,0 +1,24 @@
using System;
namespace StellaOps.Feedser.Merge.Identity;
/// <summary>
/// Normalized alias representation used within identity clusters.
/// </summary>
public sealed class AliasIdentity
{
public AliasIdentity(string value, string? scheme)
{
if (string.IsNullOrWhiteSpace(value))
{
throw new ArgumentException("Alias value must be provided.", nameof(value));
}
Value = value.Trim();
Scheme = string.IsNullOrWhiteSpace(scheme) ? null : scheme.Trim();
}
public string Value { get; }
public string? Scheme { get; }
}

View File

@@ -1,7 +1,7 @@
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|Deterministic key choice; cycle-safe.|
|Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|DONE `AdvisoryIdentityResolver` builds alias-driven clusters with canonical key selection + unit coverage.|
|Precedence policy engine|BE-Merge|Architecture|PSIRT/OVAL > NVD; CERTs enrich; KEV flag.|
|NEVRA comparer plus tests|BE-Merge (Distro WG)|Source.Distro fixtures|DONE Added Nevra parser/comparer with tilde-aware rpm ordering and unit coverage.|
|Debian EVR comparer plus tests|BE-Merge (Distro WG)|Debian fixtures|DONE DebianEvr comparer mirrors dpkg ordering with tilde/epoch handling and unit coverage.|

View File

@@ -10,6 +10,15 @@ public sealed class AffectedPackageStatusTests
[InlineData("KNOWN-NOT-AFFECTED", AffectedPackageStatusCatalog.KnownNotAffected)]
[InlineData("Under Investigation", AffectedPackageStatusCatalog.UnderInvestigation)]
[InlineData("Fixed", AffectedPackageStatusCatalog.Fixed)]
[InlineData("Known not vulnerable", AffectedPackageStatusCatalog.KnownNotAffected)]
[InlineData("Impacted", AffectedPackageStatusCatalog.Affected)]
[InlineData("Not Vulnerable", AffectedPackageStatusCatalog.NotAffected)]
[InlineData("Analysis in progress", AffectedPackageStatusCatalog.UnderInvestigation)]
[InlineData("Patch Available", AffectedPackageStatusCatalog.Fixed)]
[InlineData("Workaround available", AffectedPackageStatusCatalog.Mitigated)]
[InlineData("Does Not Apply", AffectedPackageStatusCatalog.NotApplicable)]
[InlineData("Awaiting fix", AffectedPackageStatusCatalog.Pending)]
[InlineData("TBD", AffectedPackageStatusCatalog.Unknown)]
public void Constructor_NormalizesStatus(string input, string expected)
{
var provenance = new AdvisoryProvenance("test", "status", "value", DateTimeOffset.UtcNow);
@@ -25,4 +34,42 @@ public sealed class AffectedPackageStatusTests
var provenance = new AdvisoryProvenance("test", "status", "value", DateTimeOffset.UtcNow);
Assert.Throws<ArgumentOutOfRangeException>(() => new AffectedPackageStatus("unsupported", provenance));
}
[Theory]
[InlineData("Not Impacted", AffectedPackageStatusCatalog.NotAffected)]
[InlineData("Resolved", AffectedPackageStatusCatalog.Fixed)]
[InlineData("Mitigation provided", AffectedPackageStatusCatalog.Mitigated)]
[InlineData("Out of scope", AffectedPackageStatusCatalog.NotApplicable)]
public void TryNormalize_ReturnsExpectedValue(string input, string expected)
{
Assert.True(AffectedPackageStatusCatalog.TryNormalize(input, out var normalized));
Assert.Equal(expected, normalized);
}
[Fact]
public void TryNormalize_ReturnsFalseForUnknown()
{
Assert.False(AffectedPackageStatusCatalog.TryNormalize("unsupported", out _));
}
[Fact]
public void Allowed_ReturnsCanonicalStatuses()
{
var expected = new[]
{
AffectedPackageStatusCatalog.KnownAffected,
AffectedPackageStatusCatalog.KnownNotAffected,
AffectedPackageStatusCatalog.UnderInvestigation,
AffectedPackageStatusCatalog.Fixed,
AffectedPackageStatusCatalog.FirstFixed,
AffectedPackageStatusCatalog.Mitigated,
AffectedPackageStatusCatalog.NotApplicable,
AffectedPackageStatusCatalog.Affected,
AffectedPackageStatusCatalog.NotAffected,
AffectedPackageStatusCatalog.Pending,
AffectedPackageStatusCatalog.Unknown,
};
Assert.Equal(expected, AffectedPackageStatusCatalog.Allowed);
}
}

View File

@@ -12,6 +12,14 @@ public sealed class SeverityNormalizationTests
[InlineData("Info", "informational")]
[InlineData("negligible", "none")]
[InlineData("unknown", "unknown")]
[InlineData("Sev Critical", "critical")]
[InlineData("HIGH vendor", "high")]
[InlineData("Informative", "informational")]
[InlineData("Not Applicable", "none")]
[InlineData("Undetermined", "unknown")]
[InlineData("Priority 0", "critical")]
[InlineData("Priority-2", "medium")]
[InlineData("N/A", "none")]
[InlineData("custom-level", "custom-level")]
public void Normalize_ReturnsExpectedCanonicalValue(string input, string expected)
{

View File

@@ -7,4 +7,9 @@
<ItemGroup>
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
</ItemGroup>
<ItemGroup Condition="'$(UpdateGoldens)' == 'true'">
<EnvironmentVariables Include="UPDATE_GOLDENS">
<Value>1</Value>
</EnvironmentVariables>
</ItemGroup>
</Project>

View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
namespace StellaOps.Feedser.Models;
@@ -20,7 +21,7 @@ public static class AffectedPackageStatusCatalog
public const string Pending = "pending";
public const string Unknown = "unknown";
private static readonly HashSet<string> AllowedStatuses = new(StringComparer.OrdinalIgnoreCase)
private static readonly string[] CanonicalStatuses =
{
KnownAffected,
KnownNotAffected,
@@ -35,21 +36,121 @@ public static class AffectedPackageStatusCatalog
Unknown,
};
public static IReadOnlyCollection<string> Allowed => AllowedStatuses;
private static readonly IReadOnlyList<string> AllowedStatuses = Array.AsReadOnly(CanonicalStatuses);
private static readonly IReadOnlyDictionary<string, string> StatusMap = BuildStatusMap();
public static IReadOnlyList<string> Allowed => AllowedStatuses;
public static string Normalize(string status)
{
if (string.IsNullOrWhiteSpace(status))
{
throw new ArgumentException("Status must be provided.", nameof(status));
}
var token = status.Trim().ToLowerInvariant().Replace(' ', '_').Replace('-', '_');
if (!AllowedStatuses.Contains(token))
if (!TryNormalize(status, out var normalized))
{
throw new ArgumentOutOfRangeException(nameof(status), status, "Status is not part of the allowed affected-package status glossary.");
}
return token;
return normalized;
}
public static bool TryNormalize(string? status, [NotNullWhen(true)] out string? normalized)
{
normalized = null;
if (string.IsNullOrWhiteSpace(status))
{
return false;
}
var token = Sanitize(status);
if (token.Length == 0)
{
return false;
}
if (!StatusMap.TryGetValue(token, out normalized))
{
return false;
}
return true;
}
public static bool IsAllowed(string? status)
=> TryNormalize(status, out _);
private static IReadOnlyDictionary<string, string> BuildStatusMap()
{
var map = new Dictionary<string, string>(StringComparer.Ordinal);
foreach (var status in CanonicalStatuses)
{
map[Sanitize(status)] = status;
}
Add(map, "known not vulnerable", KnownNotAffected);
Add(map, "known unaffected", KnownNotAffected);
Add(map, "known not impacted", KnownNotAffected);
Add(map, "vulnerable", Affected);
Add(map, "impacted", Affected);
Add(map, "impacting", Affected);
Add(map, "not vulnerable", NotAffected);
Add(map, "unaffected", NotAffected);
Add(map, "not impacted", NotAffected);
Add(map, "no impact", NotAffected);
Add(map, "impact free", NotAffected);
Add(map, "investigating", UnderInvestigation);
Add(map, "analysis in progress", UnderInvestigation);
Add(map, "analysis pending", UnderInvestigation);
Add(map, "patch available", Fixed);
Add(map, "fix available", Fixed);
Add(map, "patched", Fixed);
Add(map, "resolved", Fixed);
Add(map, "remediated", Fixed);
Add(map, "workaround available", Mitigated);
Add(map, "mitigation available", Mitigated);
Add(map, "mitigation provided", Mitigated);
Add(map, "not applicable", NotApplicable);
Add(map, "n/a", NotApplicable);
Add(map, "na", NotApplicable);
Add(map, "does not apply", NotApplicable);
Add(map, "out of scope", NotApplicable);
Add(map, "pending fix", Pending);
Add(map, "awaiting fix", Pending);
Add(map, "awaiting patch", Pending);
Add(map, "scheduled", Pending);
Add(map, "planned", Pending);
Add(map, "tbd", Unknown);
Add(map, "to be determined", Unknown);
Add(map, "undetermined", Unknown);
Add(map, "not yet known", Unknown);
return map;
}
private static void Add(IDictionary<string, string> map, string alias, string canonical)
{
var key = Sanitize(alias);
if (key.Length == 0)
{
return;
}
map[key] = canonical;
}
private static string Sanitize(string value)
{
var span = value.AsSpan();
var buffer = new char[span.Length];
var index = 0;
foreach (var ch in span)
{
if (char.IsLetterOrDigit(ch))
{
buffer[index++] = char.ToLowerInvariant(ch);
}
}
return index == 0 ? string.Empty : new string(buffer, 0, index);
}
}

View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace StellaOps.Feedser.Models;
@@ -8,28 +9,76 @@ namespace StellaOps.Feedser.Models;
/// </summary>
public static class SeverityNormalization
{
private static readonly IReadOnlyDictionary<string, string> SeverityMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
private static readonly IReadOnlyDictionary<string, string> SeverityMap = new Dictionary<string, string>
{
["critical"] = "critical",
["sev_critical"] = "critical",
["sev-critical"] = "critical",
["crit"] = "critical",
["sevcritical"] = "critical",
["extreme"] = "critical",
["verycritical"] = "critical",
["veryhigh"] = "critical",
["p0"] = "critical",
["priority0"] = "critical",
["high"] = "high",
["sev_high"] = "high",
["sev-high"] = "high",
["sevhigh"] = "high",
["important"] = "high",
["severe"] = "high",
["major"] = "high",
["urgent"] = "high",
["elevated"] = "high",
["p1"] = "high",
["priority1"] = "high",
["medium"] = "medium",
["moderate"] = "medium",
["normal"] = "medium",
["avg"] = "medium",
["average"] = "medium",
["standard"] = "medium",
["p2"] = "medium",
["priority2"] = "medium",
["low"] = "low",
["minor"] = "low",
["info"] = "informational",
["minimal"] = "low",
["limited"] = "low",
["p3"] = "low",
["priority3"] = "low",
["informational"] = "informational",
["info"] = "informational",
["informative"] = "informational",
["notice"] = "informational",
["advisory"] = "informational",
["none"] = "none",
["negligible"] = "none",
["insignificant"] = "none",
["notapplicable"] = "none",
["na"] = "none",
["unknown"] = "unknown",
["undetermined"] = "unknown",
["notdefined"] = "unknown",
["notspecified"] = "unknown",
["pending"] = "unknown",
["tbd"] = "unknown",
};
private static readonly char[] TokenSeparators =
{
' ',
'/',
'\\',
'-',
'_',
',',
';',
':',
'(',
')',
'[',
']',
'{',
'}',
'|',
'+',
'&'
};
public static readonly IReadOnlyCollection<string> CanonicalLevels = new[]
@@ -51,18 +100,53 @@ public static class SeverityNormalization
}
var trimmed = severity.Trim();
if (SeverityMap.TryGetValue(trimmed, out var mapped))
if (TryNormalizeToken(trimmed, out var mapped))
{
return mapped;
}
var lowered = trimmed.ToLowerInvariant();
if (SeverityMap.TryGetValue(lowered, out mapped))
foreach (var token in trimmed.Split(TokenSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
{
return mapped;
if (TryNormalizeToken(token, out mapped))
{
return mapped;
}
}
return lowered;
return trimmed.ToLowerInvariant();
}
private static bool TryNormalizeToken(string value, out string mapped)
{
var normalized = NormalizeToken(value);
if (normalized.Length == 0)
{
mapped = string.Empty;
return false;
}
if (!SeverityMap.TryGetValue(normalized, out var mappedValue))
{
mapped = string.Empty;
return false;
}
mapped = mappedValue;
return true;
}
private static string NormalizeToken(string value)
{
var builder = new StringBuilder(value.Length);
foreach (var ch in value)
{
if (char.IsLetterOrDigit(ch))
{
builder.Append(char.ToLowerInvariant(ch));
}
}
return builder.ToString();
}
}

View File

@@ -12,7 +12,7 @@
|Range primitives for SemVer/EVR/NEVRA metadata|BE-Merge|Models|TODO keep structured values without parsing logic; ensure merge/export parity.|
|Provenance envelope field masks|BE-Merge|Models|TODO guarantee traceability for each mapped field.|
|Backward-compatibility playbook|BE-Merge, QA|Models|DONE see `BACKWARD_COMPATIBILITY.md` for evolution policy/test checklist.|
|Golden canonical examples|QA|Models|DOING `CanonicalExampleFactory` + tests added; fixtures need regeneration hookup to enable easy updates.|
|Golden canonical examples|QA|Models|DONE added `/p:UpdateGoldens=true` test hook wiring `UPDATE_GOLDENS=1` so canonical fixtures regenerate via `dotnet test`; docs/tests unchanged.|
|Serialization determinism regression tests|QA|Models|TODO automate hash comparisons across locales/runs.|
|Severity normalization helpers|BE-Merge|Models|DOING helper introduced to normalize vendor severities; refine synonym coverage.|
|AffectedPackage status glossary & guardrails|BE-Merge|Models|DOING status catalog with validation added; monitor for additional vendor values.|
|Severity normalization helpers|BE-Merge|Models|DONE helper now normalizes compound vendor labels/priority tiers with expanded synonym coverage and regression tests.|
|AffectedPackage status glossary & guardrails|BE-Merge|Models|DONE catalog now exposes deterministic listing, TryNormalize helpers, and synonym coverage for vendor phrases (not vulnerable, workaround available, etc.).|

View File

@@ -91,7 +91,7 @@
"url": "https://www.cve.org/CVERecord?id=CVE-2024-9991"
}
],
"severity": "high vendor",
"severity": "high",
"summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).",
"title": "Multiple vulnerabilities in Example Gateway"
}
}

View File

@@ -22,6 +22,7 @@ public sealed class ExportStateManagerTests
cursor: "cursor-1",
targetRepository: "registry.local/json",
exporterVersion: "1.0.0",
resetBaseline: true,
cancellationToken: CancellationToken.None);
Assert.Equal("export:json", record.Id);
@@ -35,6 +36,93 @@ public sealed class ExportStateManagerTests
Assert.Equal(timeProvider.Now, record.UpdatedAt);
}
[Fact]
public async Task StoreFullExport_ResetBaselineOverridesExisting()
{
var store = new InMemoryExportStateStore();
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-20T12:00:00Z"));
var manager = new ExportStateManager(store, timeProvider);
await manager.StoreFullExportAsync(
exporterId: "export:json",
exportId: "20240720T120000Z",
exportDigest: "sha256:base",
cursor: "cursor-base",
targetRepository: null,
exporterVersion: "1.0.0",
resetBaseline: true,
cancellationToken: CancellationToken.None);
timeProvider.Advance(TimeSpan.FromMinutes(5));
var withoutReset = await manager.StoreFullExportAsync(
exporterId: "export:json",
exportId: "20240720T120500Z",
exportDigest: "sha256:new",
cursor: "cursor-new",
targetRepository: null,
exporterVersion: "1.0.1",
resetBaseline: false,
cancellationToken: CancellationToken.None);
Assert.Equal("20240720T120000Z", withoutReset.BaseExportId);
Assert.Equal("sha256:base", withoutReset.BaseDigest);
Assert.Equal("sha256:new", withoutReset.LastFullDigest);
Assert.Equal("cursor-new", withoutReset.ExportCursor);
Assert.Equal(timeProvider.Now, withoutReset.UpdatedAt);
timeProvider.Advance(TimeSpan.FromMinutes(5));
var reset = await manager.StoreFullExportAsync(
exporterId: "export:json",
exportId: "20240720T121000Z",
exportDigest: "sha256:final",
cursor: "cursor-final",
targetRepository: null,
exporterVersion: "1.0.2",
resetBaseline: true,
cancellationToken: CancellationToken.None);
Assert.Equal("20240720T121000Z", reset.BaseExportId);
Assert.Equal("sha256:final", reset.BaseDigest);
Assert.Equal("sha256:final", reset.LastFullDigest);
Assert.Null(reset.LastDeltaDigest);
Assert.Equal("cursor-final", reset.ExportCursor);
Assert.Equal(timeProvider.Now, reset.UpdatedAt);
}
[Fact]
public async Task StoreFullExport_ResetsBaselineWhenRepositoryChanges()
{
var store = new InMemoryExportStateStore();
var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-21T08:00:00Z"));
var manager = new ExportStateManager(store, timeProvider);
await manager.StoreFullExportAsync(
exporterId: "export:json",
exportId: "20240721T080000Z",
exportDigest: "sha256:base",
cursor: "cursor-base",
targetRepository: "registry/v1/json",
exporterVersion: "1.0.0",
resetBaseline: true,
cancellationToken: CancellationToken.None);
timeProvider.Advance(TimeSpan.FromMinutes(10));
var updated = await manager.StoreFullExportAsync(
exporterId: "export:json",
exportId: "20240721T081000Z",
exportDigest: "sha256:new",
cursor: "cursor-new",
targetRepository: "registry/v2/json",
exporterVersion: "1.1.0",
resetBaseline: false,
cancellationToken: CancellationToken.None);
Assert.Equal("20240721T081000Z", updated.BaseExportId);
Assert.Equal("sha256:new", updated.BaseDigest);
Assert.Equal("sha256:new", updated.LastFullDigest);
Assert.Equal("registry/v2/json", updated.TargetRepository);
}
[Fact]
public async Task StoreDeltaExportRequiresBaseline()
{
@@ -63,6 +151,7 @@ public sealed class ExportStateManagerTests
cursor: "cursor-1",
targetRepository: null,
exporterVersion: "1.0.0",
resetBaseline: true,
cancellationToken: CancellationToken.None);
timeProvider.Advance(TimeSpan.FromMinutes(10));

View File

@@ -31,6 +31,7 @@ public sealed class ExportStateManager
string? cursor,
string? targetRepository,
string exporterVersion,
bool resetBaseline,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrEmpty(exporterId);
@@ -39,36 +40,59 @@ public sealed class ExportStateManager
ArgumentException.ThrowIfNullOrEmpty(exporterVersion);
var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false);
var repository = string.IsNullOrWhiteSpace(targetRepository) ? existing?.TargetRepository : targetRepository;
var now = _timeProvider.GetUtcNow();
var baseExportId = existing?.BaseExportId ?? exportId;
var baseDigest = existing?.BaseDigest ?? exportDigest;
if (existing is null)
{
var resolvedRepository = string.IsNullOrWhiteSpace(targetRepository) ? null : targetRepository;
return await _store.UpsertAsync(
new ExportStateRecord(
exporterId,
BaseExportId: exportId,
BaseDigest: exportDigest,
LastFullDigest: exportDigest,
LastDeltaDigest: null,
ExportCursor: cursor ?? exportDigest,
TargetRepository: resolvedRepository,
ExporterVersion: exporterVersion,
UpdatedAt: now),
cancellationToken).ConfigureAwait(false);
}
var record = existing is null
? new ExportStateRecord(
exporterId,
baseExportId,
baseDigest,
exportDigest,
LastDeltaDigest: null,
ExportCursor: cursor ?? exportDigest,
TargetRepository: repository,
ExporterVersion: exporterVersion,
UpdatedAt: now)
var repositorySpecified = !string.IsNullOrWhiteSpace(targetRepository);
var resolvedRepo = repositorySpecified ? targetRepository : existing.TargetRepository;
var repositoryChanged = repositorySpecified
&& !string.Equals(existing.TargetRepository, targetRepository, StringComparison.Ordinal);
var shouldResetBaseline =
resetBaseline
|| string.IsNullOrWhiteSpace(existing.BaseExportId)
|| string.IsNullOrWhiteSpace(existing.BaseDigest)
|| repositoryChanged;
var updatedRecord = shouldResetBaseline
? existing with
{
BaseExportId = exportId,
BaseDigest = exportDigest,
LastFullDigest = exportDigest,
LastDeltaDigest = null,
ExportCursor = cursor ?? exportDigest,
TargetRepository = resolvedRepo,
ExporterVersion = exporterVersion,
UpdatedAt = now,
}
: existing with
{
BaseExportId = baseExportId,
BaseDigest = baseDigest,
LastFullDigest = exportDigest,
LastDeltaDigest = null,
ExportCursor = cursor ?? existing.ExportCursor,
TargetRepository = repository,
TargetRepository = resolvedRepo,
ExporterVersion = exporterVersion,
UpdatedAt = now,
};
return await _store.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
return await _store.UpsertAsync(updatedRecord, cancellationToken).ConfigureAwait(false);
}
public async Task<ExportStateRecord> StoreDeltaExportAsync(