diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs b/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs index 56bf1cc6..c636457c 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs +++ b/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs @@ -71,6 +71,17 @@ public sealed class JsonFeedExporter : IFeedExporter return; } + var resetBaseline = existingState is null + || string.IsNullOrWhiteSpace(existingState.BaseExportId) + || string.IsNullOrWhiteSpace(existingState.BaseDigest); + + if (existingState is not null + && !string.IsNullOrWhiteSpace(_options.TargetRepository) + && !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal)) + { + resetBaseline = true; + } + await _stateManager.StoreFullExportAsync( ExporterId, exportId, @@ -78,6 +89,7 @@ public sealed class JsonFeedExporter : IFeedExporter cursor: digest, targetRepository: _options.TargetRepository, exporterVersion: _exporterVersion, + resetBaseline: resetBaseline, cancellationToken: cancellationToken).ConfigureAwait(false); await JsonExportManifestWriter.WriteAsync(result, digest, _exporterVersion, cancellationToken).ConfigureAwait(false); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs index 59314c02..59de7ddb 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs @@ -16,6 +16,7 @@ public sealed class TrivyDbExportPlannerTests Assert.Equal("sha256:abcd", plan.TreeDigest); Assert.Null(plan.BaseExportId); Assert.Null(plan.BaseManifestDigest); + Assert.True(plan.ResetBaseline); } [Fact] @@ -39,6 +40,7 @@ public sealed class TrivyDbExportPlannerTests Assert.Equal("sha256:unchanged", plan.TreeDigest); Assert.Equal("20240810T000000Z", plan.BaseExportId); Assert.Equal("sha256:base", plan.BaseManifestDigest); + Assert.False(plan.ResetBaseline); } [Fact] @@ -62,5 +64,12 @@ public sealed class TrivyDbExportPlannerTests Assert.Equal("sha256:new", plan.TreeDigest); Assert.Equal("20240810T000000Z", plan.BaseExportId); Assert.Equal("sha256:base", plan.BaseManifestDigest); + Assert.False(plan.ResetBaseline); + + var deltaState = state with { LastDeltaDigest = "sha256:delta" }; + var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer"); + + Assert.Equal(TrivyDbExportMode.Full, deltaPlan.Mode); + Assert.True(deltaPlan.ResetBaseline); } } diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs index 5bf0aa61..befce26b 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs @@ -281,6 +281,77 @@ public sealed class TrivyDbFeedExporterTests : IDisposable Assert.Empty(orasPusher.Pushes); } + [Fact] + public async Task ExportAsync_ResetsBaselineWhenDeltaChainExists() + { + var advisory = CreateSampleAdvisory("CVE-2024-5000", "Baseline reset"); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + TargetRepository = "registry.example/trivy", + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-22T00:00:00Z", CultureInfo.InvariantCulture)); + var existingRecord = new ExportStateRecord( + TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240919T120000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: "sha256:delta", + ExportCursor: "sha256:old", + TargetRepository: "registry.example/trivy", + ExporterVersion: "0.9.0", + UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30)); + await stateStore.UpsertAsync(existingRecord, CancellationToken.None); + + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-23T00:00:00Z", + UpdatedAt = "2024-09-22T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var updated = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(updated); + Assert.Equal("20240922T000000Z", updated!.BaseExportId); + Assert.Equal(updated.BaseDigest, updated.LastFullDigest); + Assert.Null(updated.LastDeltaDigest); + Assert.NotEqual("sha256:old", updated.ExportCursor); + Assert.Equal("registry.example/trivy", updated.TargetRepository); + } + private static Advisory CreateSampleAdvisory( string advisoryKey = "CVE-2024-9999", string title = "Trivy Export Test") diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md b/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md index 4f7a7c4e..dca12c7c 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md @@ -8,6 +8,6 @@ |Offline bundle toggle|BE-Export|Exporters|DONE – Deterministic OCI layout bundle emitted when enabled.| |Deterministic ordering of advisories|BE-Export|Models|DONE – exporter now loads advisories, sorts by advisoryKey, and emits sorted JSON trees with deterministic OCI payloads.| |End-to-end tests with small dataset|QA|Exporters|DONE – added deterministic round-trip test covering OCI layout, media types, and digest stability w/ repeated inputs.| -|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DOING – `ExportStateManager` keeps stable base export metadata; delta reset remains pending.| +|ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE – baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.| |Streamed package building to avoid large copies|BE-Export|Exporters|DONE – metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.| |Plan incremental/delta exports|BE-Export|Exporters|TODO – design reuse of existing blobs/layers when inputs unchanged instead of rewriting full trees each run.| diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs index d889eae1..02ee9cad 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs @@ -4,4 +4,5 @@ public sealed record TrivyDbExportPlan( TrivyDbExportMode Mode, string TreeDigest, string? BaseExportId, - string? BaseManifestDigest); + string? BaseManifestDigest, + bool ResetBaseline); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs index 0f9fb0c2..330902de 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs @@ -11,7 +11,12 @@ public sealed class TrivyDbExportPlanner if (existingState is null) { - return new TrivyDbExportPlan(TrivyDbExportMode.Full, treeDigest, BaseExportId: null, BaseManifestDigest: null); + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + BaseExportId: null, + BaseManifestDigest: null, + ResetBaseline: true); } if (string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal)) @@ -20,14 +25,18 @@ public sealed class TrivyDbExportPlanner TrivyDbExportMode.Skip, treeDigest, existingState.BaseExportId, - existingState.LastFullDigest); + existingState.LastFullDigest, + ResetBaseline: false); } + var resetBaseline = existingState.LastDeltaDigest is not null; + // Placeholder for future delta support – current behavior always rebuilds when tree changes. return new TrivyDbExportPlan( TrivyDbExportMode.Full, treeDigest, existingState.BaseExportId, - existingState.LastFullDigest); + existingState.LastFullDigest, + resetBaseline); } } diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs index 773b72c8..9b0258d9 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs +++ b/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs @@ -130,6 +130,18 @@ public sealed class TrivyDbFeedExporter : IFeedExporter exportId, ociResult.ManifestDigest); + var resetBaseline = plan.ResetBaseline + || existingState is null + || string.IsNullOrWhiteSpace(existingState.BaseExportId) + || string.IsNullOrWhiteSpace(existingState.BaseDigest); + + if (existingState is not null + && !string.IsNullOrWhiteSpace(_options.TargetRepository) + && !string.Equals(existingState.TargetRepository, _options.TargetRepository, StringComparison.Ordinal)) + { + resetBaseline = true; + } + await _stateManager.StoreFullExportAsync( ExporterId, exportId, @@ -137,6 +149,7 @@ public sealed class TrivyDbFeedExporter : IFeedExporter cursor: treeDigest, targetRepository: _options.TargetRepository, exporterVersion: _exporterVersion, + resetBaseline: resetBaseline, cancellationToken: cancellationToken).ConfigureAwait(false); await CreateOfflineBundleAsync(destination, exportId, exportedAt, cancellationToken).ConfigureAwait(false); diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs b/src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs new file mode 100644 index 00000000..2bc570f0 --- /dev/null +++ b/src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs @@ -0,0 +1,91 @@ +using System; +using System.Linq; +using StellaOps.Feedser.Merge.Identity; +using StellaOps.Feedser.Models; +using Xunit; + +namespace StellaOps.Feedser.Merge.Tests; + +public sealed class AdvisoryIdentityResolverTests +{ + private readonly AdvisoryIdentityResolver _resolver = new(); + + [Fact] + public void Resolve_GroupsBySharedCveAlias() + { + var nvd = CreateAdvisory("CVE-2025-1234", aliases: new[] { "CVE-2025-1234" }, source: "nvd"); + var vendor = CreateAdvisory("VSA-2025-01", aliases: new[] { "CVE-2025-1234", "VSA-2025-01" }, source: "vendor"); + + var clusters = _resolver.Resolve(new[] { nvd, vendor }); + + var cluster = Assert.Single(clusters); + Assert.Equal("CVE-2025-1234", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "CVE-2025-1234")); + } + + [Fact] + public void Resolve_PrefersPsirtAliasWhenNoCve() + { + var vendor = CreateAdvisory("VMSA-2025-0001", aliases: new[] { "VMSA-2025-0001" }, source: "vmware"); + var osv = CreateAdvisory("OSV-2025-1", aliases: new[] { "OSV-2025-1", "GHSA-xxxx-yyyy-zzzz", "VMSA-2025-0001" }, source: "osv"); + + var clusters = _resolver.Resolve(new[] { vendor, osv }); + + var cluster = Assert.Single(clusters); + Assert.Equal("VMSA-2025-0001", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "VMSA-2025-0001")); + } + + [Fact] + public void Resolve_FallsBackToGhsaWhenOnlyGhsaPresent() + { + var ghsa = CreateAdvisory("GHSA-aaaa-bbbb-cccc", aliases: new[] { "GHSA-aaaa-bbbb-cccc" }, source: "ghsa"); + var osv = CreateAdvisory("OSV-2025-99", aliases: new[] { "OSV-2025-99", "GHSA-aaaa-bbbb-cccc" }, source: "osv"); + + var clusters = _resolver.Resolve(new[] { ghsa, osv }); + + var cluster = Assert.Single(clusters); + Assert.Equal("GHSA-aaaa-bbbb-cccc", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "GHSA-aaaa-bbbb-cccc")); + } + + [Fact] + public void Resolve_GroupsByKeyWhenNoAliases() + { + var first = CreateAdvisory("custom-1", aliases: Array.Empty(), source: "source-a"); + var second = CreateAdvisory("custom-1", aliases: Array.Empty(), source: "source-b"); + + var clusters = _resolver.Resolve(new[] { first, second }); + + var cluster = Assert.Single(clusters); + Assert.Equal("custom-1", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.Contains(cluster.Aliases, alias => alias.Value == "custom-1"); + } + + private static Advisory CreateAdvisory(string key, string[] aliases, string source) + { + var provenance = new[] + { + new AdvisoryProvenance(source, "mapping", key, DateTimeOffset.UtcNow), + }; + + return new Advisory( + key, + $"{key} title", + $"{key} summary", + "en", + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow, + null, + exploitKnown: false, + aliases, + Array.Empty(), + Array.Empty(), + Array.Empty(), + provenance); + } +} diff --git a/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs b/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs new file mode 100644 index 00000000..97dd7dee --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Identity; + +/// +/// Represents a connected component of advisories that refer to the same vulnerability. +/// +public sealed class AdvisoryIdentityCluster +{ + public AdvisoryIdentityCluster(string advisoryKey, IEnumerable advisories, IEnumerable aliases) + { + AdvisoryKey = !string.IsNullOrWhiteSpace(advisoryKey) + ? advisoryKey.Trim() + : throw new ArgumentException("Canonical advisory key must be provided.", nameof(advisoryKey)); + + var advisoriesArray = (advisories ?? throw new ArgumentNullException(nameof(advisories))) + .Where(static advisory => advisory is not null) + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ThenBy(static advisory => advisory.Provenance.Length) + .ThenBy(static advisory => advisory.Title, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + if (advisoriesArray.IsDefaultOrEmpty) + { + throw new ArgumentException("At least one advisory is required for a cluster.", nameof(advisories)); + } + + var aliasArray = (aliases ?? throw new ArgumentNullException(nameof(aliases))) + .Where(static alias => alias is not null && !string.IsNullOrWhiteSpace(alias.Value)) + .GroupBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase) + .Select(static group => + { + var representative = group + .OrderBy(static entry => entry.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static entry => entry.Value, StringComparer.OrdinalIgnoreCase) + .First(); + return representative; + }) + .OrderBy(static alias => alias.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + Advisories = advisoriesArray; + Aliases = aliasArray; + } + + public string AdvisoryKey { get; } + + public ImmutableArray Advisories { get; } + + public ImmutableArray Aliases { get; } +} diff --git a/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs b/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs new file mode 100644 index 00000000..52c94599 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs @@ -0,0 +1,303 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Runtime.CompilerServices; +using StellaOps.Feedser.Models; + +namespace StellaOps.Feedser.Merge.Identity; + +/// +/// Builds an alias-driven identity graph that groups advisories referring to the same vulnerability. +/// +public sealed class AdvisoryIdentityResolver +{ + private static readonly string[] CanonicalAliasPriority = + { + AliasSchemes.Cve, + AliasSchemes.Rhsa, + AliasSchemes.Usn, + AliasSchemes.Dsa, + AliasSchemes.SuseSu, + AliasSchemes.Msrc, + AliasSchemes.CiscoSa, + AliasSchemes.OracleCpu, + AliasSchemes.Vmsa, + AliasSchemes.Apsb, + AliasSchemes.Apa, + AliasSchemes.AppleHt, + AliasSchemes.ChromiumPost, + AliasSchemes.Icsa, + AliasSchemes.Jvndb, + AliasSchemes.Jvn, + AliasSchemes.Bdu, + AliasSchemes.Vu, + AliasSchemes.Ghsa, + AliasSchemes.OsV, + }; + + /// + /// Groups the provided advisories into identity clusters using normalized aliases. + /// + public IReadOnlyList Resolve(IEnumerable advisories) + { + ArgumentNullException.ThrowIfNull(advisories); + + var materialized = advisories + .Where(static advisory => advisory is not null) + .Distinct() + .ToArray(); + + if (materialized.Length == 0) + { + return Array.Empty(); + } + + var aliasIndex = BuildAliasIndex(materialized); + var visited = new HashSet(); + var clusters = new List(); + + foreach (var advisory in materialized) + { + if (!visited.Add(advisory)) + { + continue; + } + + var component = TraverseComponent(advisory, visited, aliasIndex); + var key = DetermineCanonicalKey(component); + var aliases = component + .SelectMany(static entry => entry.Aliases) + .Select(static alias => new AliasIdentity(alias.Normalized, alias.Scheme)); + clusters.Add(new AdvisoryIdentityCluster(key, component.Select(static entry => entry.Advisory), aliases)); + } + + return clusters + .OrderBy(static cluster => cluster.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static Dictionary> BuildAliasIndex(IEnumerable advisories) + { + var index = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var advisory in advisories) + { + foreach (var alias in ExtractAliases(advisory)) + { + if (!index.TryGetValue(alias.Normalized, out var list)) + { + list = new List(); + index[alias.Normalized] = list; + } + + list.Add(new AdvisoryAliasEntry(advisory, alias.Normalized, alias.Scheme)); + } + } + + return index; + } + + private static IReadOnlyList TraverseComponent( + Advisory root, + HashSet visited, + Dictionary> aliasIndex) + { + var stack = new Stack(); + stack.Push(root); + + var bindings = new Dictionary(ReferenceEqualityComparer.Instance); + + while (stack.Count > 0) + { + var advisory = stack.Pop(); + + if (!bindings.TryGetValue(advisory, out var binding)) + { + binding = new AliasBinding(advisory); + bindings[advisory] = binding; + } + + foreach (var alias in ExtractAliases(advisory)) + { + binding.AddAlias(alias.Normalized, alias.Scheme); + + if (!aliasIndex.TryGetValue(alias.Normalized, out var neighbors)) + { + continue; + } + + foreach (var neighbor in neighbors.Select(static entry => entry.Advisory)) + { + if (visited.Add(neighbor)) + { + stack.Push(neighbor); + } + + if (!bindings.TryGetValue(neighbor, out var neighborBinding)) + { + neighborBinding = new AliasBinding(neighbor); + bindings[neighbor] = neighborBinding; + } + + neighborBinding.AddAlias(alias.Normalized, alias.Scheme); + } + } + } + + return bindings.Values + .OrderBy(static binding => binding.Advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static string DetermineCanonicalKey(IReadOnlyList component) + { + var aliases = component + .SelectMany(static binding => binding.Aliases) + .Where(static alias => !string.IsNullOrWhiteSpace(alias.Normalized)) + .ToArray(); + + foreach (var scheme in CanonicalAliasPriority) + { + var candidate = aliases + .Where(alias => string.Equals(alias.Scheme, scheme, StringComparison.OrdinalIgnoreCase)) + .Select(static alias => alias.Normalized) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (candidate is not null) + { + return candidate; + } + } + + var fallbackAlias = aliases + .Select(static alias => alias.Normalized) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (!string.IsNullOrWhiteSpace(fallbackAlias)) + { + return fallbackAlias; + } + + var advisoryKey = component + .Select(static binding => binding.Advisory.AdvisoryKey) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (!string.IsNullOrWhiteSpace(advisoryKey)) + { + return advisoryKey.Trim(); + } + + throw new InvalidOperationException("Unable to determine canonical advisory key for cluster."); + } + + private static IEnumerable ExtractAliases(Advisory advisory) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var candidate in EnumerateAliasCandidates(advisory)) + { + if (string.IsNullOrWhiteSpace(candidate)) + { + continue; + } + + var trimmed = candidate.Trim(); + if (!seen.Add(trimmed)) + { + continue; + } + + if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out var scheme) && + !string.IsNullOrWhiteSpace(normalized)) + { + yield return new AliasProjection(normalized.Trim(), string.IsNullOrWhiteSpace(scheme) ? null : scheme); + } + else if (!string.IsNullOrWhiteSpace(normalized)) + { + yield return new AliasProjection(normalized.Trim(), null); + } + } + } + + private static IEnumerable EnumerateAliasCandidates(Advisory advisory) + { + if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) + { + yield return advisory.AdvisoryKey; + } + + if (!advisory.Aliases.IsDefaultOrEmpty) + { + foreach (var alias in advisory.Aliases) + { + if (!string.IsNullOrWhiteSpace(alias)) + { + yield return alias; + } + } + } + } + + private readonly record struct AdvisoryAliasEntry(Advisory Advisory, string Normalized, string? Scheme); + + private readonly record struct AliasProjection(string Normalized, string? Scheme); + + private sealed class AliasBinding + { + private readonly HashSet _aliases = new(HashSetAliasComparer.Instance); + + public AliasBinding(Advisory advisory) + { + Advisory = advisory ?? throw new ArgumentNullException(nameof(advisory)); + } + + public Advisory Advisory { get; } + + public IReadOnlyCollection Aliases => _aliases; + + public void AddAlias(string normalized, string? scheme) + { + if (string.IsNullOrWhiteSpace(normalized)) + { + return; + } + + _aliases.Add(new AliasProjection(normalized.Trim(), scheme is null ? null : scheme.Trim())); + } + } + + private sealed class HashSetAliasComparer : IEqualityComparer + { + public static readonly HashSetAliasComparer Instance = new(); + + public bool Equals(AliasProjection x, AliasProjection y) + => string.Equals(x.Normalized, y.Normalized, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Scheme, y.Scheme, StringComparison.OrdinalIgnoreCase); + + public int GetHashCode(AliasProjection obj) + { + var hash = StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Normalized); + if (!string.IsNullOrWhiteSpace(obj.Scheme)) + { + hash = HashCode.Combine(hash, StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Scheme)); + } + + return hash; + } + } + + private sealed class ReferenceEqualityComparer : IEqualityComparer + where T : class + { + public static readonly ReferenceEqualityComparer Instance = new(); + + public bool Equals(T? x, T? y) => ReferenceEquals(x, y); + + public int GetHashCode(T obj) => RuntimeHelpers.GetHashCode(obj); + } +} diff --git a/src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs b/src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs new file mode 100644 index 00000000..193f7cc2 --- /dev/null +++ b/src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs @@ -0,0 +1,24 @@ +using System; + +namespace StellaOps.Feedser.Merge.Identity; + +/// +/// Normalized alias representation used within identity clusters. +/// +public sealed class AliasIdentity +{ + public AliasIdentity(string value, string? scheme) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Alias value must be provided.", nameof(value)); + } + + Value = value.Trim(); + Scheme = string.IsNullOrWhiteSpace(scheme) ? null : scheme.Trim(); + } + + public string Value { get; } + + public string? Scheme { get; } +} diff --git a/src/StellaOps.Feedser.Merge/TASKS.md b/src/StellaOps.Feedser.Merge/TASKS.md index 37122944..6969e339 100644 --- a/src/StellaOps.Feedser.Merge/TASKS.md +++ b/src/StellaOps.Feedser.Merge/TASKS.md @@ -1,7 +1,7 @@ # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| -|Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|Deterministic key choice; cycle-safe.| +|Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|DONE – `AdvisoryIdentityResolver` builds alias-driven clusters with canonical key selection + unit coverage.| |Precedence policy engine|BE-Merge|Architecture|PSIRT/OVAL > NVD; CERTs enrich; KEV flag.| |NEVRA comparer plus tests|BE-Merge (Distro WG)|Source.Distro fixtures|DONE – Added Nevra parser/comparer with tilde-aware rpm ordering and unit coverage.| |Debian EVR comparer plus tests|BE-Merge (Distro WG)|Debian fixtures|DONE – DebianEvr comparer mirrors dpkg ordering with tilde/epoch handling and unit coverage.| diff --git a/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs b/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs index e90401cf..858946ad 100644 --- a/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs +++ b/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs @@ -10,6 +10,15 @@ public sealed class AffectedPackageStatusTests [InlineData("KNOWN-NOT-AFFECTED", AffectedPackageStatusCatalog.KnownNotAffected)] [InlineData("Under Investigation", AffectedPackageStatusCatalog.UnderInvestigation)] [InlineData("Fixed", AffectedPackageStatusCatalog.Fixed)] + [InlineData("Known not vulnerable", AffectedPackageStatusCatalog.KnownNotAffected)] + [InlineData("Impacted", AffectedPackageStatusCatalog.Affected)] + [InlineData("Not Vulnerable", AffectedPackageStatusCatalog.NotAffected)] + [InlineData("Analysis in progress", AffectedPackageStatusCatalog.UnderInvestigation)] + [InlineData("Patch Available", AffectedPackageStatusCatalog.Fixed)] + [InlineData("Workaround available", AffectedPackageStatusCatalog.Mitigated)] + [InlineData("Does Not Apply", AffectedPackageStatusCatalog.NotApplicable)] + [InlineData("Awaiting fix", AffectedPackageStatusCatalog.Pending)] + [InlineData("TBD", AffectedPackageStatusCatalog.Unknown)] public void Constructor_NormalizesStatus(string input, string expected) { var provenance = new AdvisoryProvenance("test", "status", "value", DateTimeOffset.UtcNow); @@ -25,4 +34,42 @@ public sealed class AffectedPackageStatusTests var provenance = new AdvisoryProvenance("test", "status", "value", DateTimeOffset.UtcNow); Assert.Throws(() => new AffectedPackageStatus("unsupported", provenance)); } + + [Theory] + [InlineData("Not Impacted", AffectedPackageStatusCatalog.NotAffected)] + [InlineData("Resolved", AffectedPackageStatusCatalog.Fixed)] + [InlineData("Mitigation provided", AffectedPackageStatusCatalog.Mitigated)] + [InlineData("Out of scope", AffectedPackageStatusCatalog.NotApplicable)] + public void TryNormalize_ReturnsExpectedValue(string input, string expected) + { + Assert.True(AffectedPackageStatusCatalog.TryNormalize(input, out var normalized)); + Assert.Equal(expected, normalized); + } + + [Fact] + public void TryNormalize_ReturnsFalseForUnknown() + { + Assert.False(AffectedPackageStatusCatalog.TryNormalize("unsupported", out _)); + } + + [Fact] + public void Allowed_ReturnsCanonicalStatuses() + { + var expected = new[] + { + AffectedPackageStatusCatalog.KnownAffected, + AffectedPackageStatusCatalog.KnownNotAffected, + AffectedPackageStatusCatalog.UnderInvestigation, + AffectedPackageStatusCatalog.Fixed, + AffectedPackageStatusCatalog.FirstFixed, + AffectedPackageStatusCatalog.Mitigated, + AffectedPackageStatusCatalog.NotApplicable, + AffectedPackageStatusCatalog.Affected, + AffectedPackageStatusCatalog.NotAffected, + AffectedPackageStatusCatalog.Pending, + AffectedPackageStatusCatalog.Unknown, + }; + + Assert.Equal(expected, AffectedPackageStatusCatalog.Allowed); + } } diff --git a/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs b/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs index 7b9099e4..43fa9108 100644 --- a/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs +++ b/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs @@ -12,6 +12,14 @@ public sealed class SeverityNormalizationTests [InlineData("Info", "informational")] [InlineData("negligible", "none")] [InlineData("unknown", "unknown")] + [InlineData("Sev Critical", "critical")] + [InlineData("HIGH vendor", "high")] + [InlineData("Informative", "informational")] + [InlineData("Not Applicable", "none")] + [InlineData("Undetermined", "unknown")] + [InlineData("Priority 0", "critical")] + [InlineData("Priority-2", "medium")] + [InlineData("N/A", "none")] [InlineData("custom-level", "custom-level")] public void Normalize_ReturnsExpectedCanonicalValue(string input, string expected) { diff --git a/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj b/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj index 6d0b9223..0d7bd8f5 100644 --- a/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj +++ b/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj @@ -7,4 +7,9 @@ + + + 1 + + diff --git a/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs b/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs index a5a232f9..18ad4ad8 100644 --- a/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs +++ b/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace StellaOps.Feedser.Models; @@ -20,7 +21,7 @@ public static class AffectedPackageStatusCatalog public const string Pending = "pending"; public const string Unknown = "unknown"; - private static readonly HashSet AllowedStatuses = new(StringComparer.OrdinalIgnoreCase) + private static readonly string[] CanonicalStatuses = { KnownAffected, KnownNotAffected, @@ -35,21 +36,121 @@ public static class AffectedPackageStatusCatalog Unknown, }; - public static IReadOnlyCollection Allowed => AllowedStatuses; + private static readonly IReadOnlyList AllowedStatuses = Array.AsReadOnly(CanonicalStatuses); + + private static readonly IReadOnlyDictionary StatusMap = BuildStatusMap(); + + public static IReadOnlyList Allowed => AllowedStatuses; public static string Normalize(string status) { - if (string.IsNullOrWhiteSpace(status)) - { - throw new ArgumentException("Status must be provided.", nameof(status)); - } - - var token = status.Trim().ToLowerInvariant().Replace(' ', '_').Replace('-', '_'); - if (!AllowedStatuses.Contains(token)) + if (!TryNormalize(status, out var normalized)) { throw new ArgumentOutOfRangeException(nameof(status), status, "Status is not part of the allowed affected-package status glossary."); } - return token; + return normalized; + } + + public static bool TryNormalize(string? status, [NotNullWhen(true)] out string? normalized) + { + normalized = null; + + if (string.IsNullOrWhiteSpace(status)) + { + return false; + } + + var token = Sanitize(status); + if (token.Length == 0) + { + return false; + } + + if (!StatusMap.TryGetValue(token, out normalized)) + { + return false; + } + + return true; + } + + public static bool IsAllowed(string? status) + => TryNormalize(status, out _); + + private static IReadOnlyDictionary BuildStatusMap() + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var status in CanonicalStatuses) + { + map[Sanitize(status)] = status; + } + + Add(map, "known not vulnerable", KnownNotAffected); + Add(map, "known unaffected", KnownNotAffected); + Add(map, "known not impacted", KnownNotAffected); + Add(map, "vulnerable", Affected); + Add(map, "impacted", Affected); + Add(map, "impacting", Affected); + Add(map, "not vulnerable", NotAffected); + Add(map, "unaffected", NotAffected); + Add(map, "not impacted", NotAffected); + Add(map, "no impact", NotAffected); + Add(map, "impact free", NotAffected); + Add(map, "investigating", UnderInvestigation); + Add(map, "analysis in progress", UnderInvestigation); + Add(map, "analysis pending", UnderInvestigation); + Add(map, "patch available", Fixed); + Add(map, "fix available", Fixed); + Add(map, "patched", Fixed); + Add(map, "resolved", Fixed); + Add(map, "remediated", Fixed); + Add(map, "workaround available", Mitigated); + Add(map, "mitigation available", Mitigated); + Add(map, "mitigation provided", Mitigated); + Add(map, "not applicable", NotApplicable); + Add(map, "n/a", NotApplicable); + Add(map, "na", NotApplicable); + Add(map, "does not apply", NotApplicable); + Add(map, "out of scope", NotApplicable); + Add(map, "pending fix", Pending); + Add(map, "awaiting fix", Pending); + Add(map, "awaiting patch", Pending); + Add(map, "scheduled", Pending); + Add(map, "planned", Pending); + Add(map, "tbd", Unknown); + Add(map, "to be determined", Unknown); + Add(map, "undetermined", Unknown); + Add(map, "not yet known", Unknown); + + return map; + } + + private static void Add(IDictionary map, string alias, string canonical) + { + var key = Sanitize(alias); + if (key.Length == 0) + { + return; + } + + map[key] = canonical; + } + + private static string Sanitize(string value) + { + var span = value.AsSpan(); + var buffer = new char[span.Length]; + var index = 0; + + foreach (var ch in span) + { + if (char.IsLetterOrDigit(ch)) + { + buffer[index++] = char.ToLowerInvariant(ch); + } + } + + return index == 0 ? string.Empty : new string(buffer, 0, index); } } diff --git a/src/StellaOps.Feedser.Models/SeverityNormalization.cs b/src/StellaOps.Feedser.Models/SeverityNormalization.cs index 7c2250e1..36340e83 100644 --- a/src/StellaOps.Feedser.Models/SeverityNormalization.cs +++ b/src/StellaOps.Feedser.Models/SeverityNormalization.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Text; namespace StellaOps.Feedser.Models; @@ -8,28 +9,76 @@ namespace StellaOps.Feedser.Models; /// public static class SeverityNormalization { - private static readonly IReadOnlyDictionary SeverityMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + private static readonly IReadOnlyDictionary SeverityMap = new Dictionary { ["critical"] = "critical", - ["sev_critical"] = "critical", - ["sev-critical"] = "critical", + ["crit"] = "critical", + ["sevcritical"] = "critical", + ["extreme"] = "critical", + ["verycritical"] = "critical", + ["veryhigh"] = "critical", + ["p0"] = "critical", + ["priority0"] = "critical", ["high"] = "high", - ["sev_high"] = "high", - ["sev-high"] = "high", + ["sevhigh"] = "high", ["important"] = "high", ["severe"] = "high", + ["major"] = "high", + ["urgent"] = "high", + ["elevated"] = "high", + ["p1"] = "high", + ["priority1"] = "high", ["medium"] = "medium", ["moderate"] = "medium", ["normal"] = "medium", ["avg"] = "medium", + ["average"] = "medium", + ["standard"] = "medium", + ["p2"] = "medium", + ["priority2"] = "medium", ["low"] = "low", ["minor"] = "low", - ["info"] = "informational", + ["minimal"] = "low", + ["limited"] = "low", + ["p3"] = "low", + ["priority3"] = "low", ["informational"] = "informational", + ["info"] = "informational", + ["informative"] = "informational", ["notice"] = "informational", + ["advisory"] = "informational", ["none"] = "none", ["negligible"] = "none", + ["insignificant"] = "none", + ["notapplicable"] = "none", + ["na"] = "none", ["unknown"] = "unknown", + ["undetermined"] = "unknown", + ["notdefined"] = "unknown", + ["notspecified"] = "unknown", + ["pending"] = "unknown", + ["tbd"] = "unknown", + }; + + private static readonly char[] TokenSeparators = + { + ' ', + '/', + '\\', + '-', + '_', + ',', + ';', + ':', + '(', + ')', + '[', + ']', + '{', + '}', + '|', + '+', + '&' }; public static readonly IReadOnlyCollection CanonicalLevels = new[] @@ -51,18 +100,53 @@ public static class SeverityNormalization } var trimmed = severity.Trim(); - if (SeverityMap.TryGetValue(trimmed, out var mapped)) + + if (TryNormalizeToken(trimmed, out var mapped)) { return mapped; } - var lowered = trimmed.ToLowerInvariant(); - - if (SeverityMap.TryGetValue(lowered, out mapped)) + foreach (var token in trimmed.Split(TokenSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) { - return mapped; + if (TryNormalizeToken(token, out mapped)) + { + return mapped; + } } - return lowered; + return trimmed.ToLowerInvariant(); + } + + private static bool TryNormalizeToken(string value, out string mapped) + { + var normalized = NormalizeToken(value); + if (normalized.Length == 0) + { + mapped = string.Empty; + return false; + } + + if (!SeverityMap.TryGetValue(normalized, out var mappedValue)) + { + mapped = string.Empty; + return false; + } + + mapped = mappedValue; + return true; + } + + private static string NormalizeToken(string value) + { + var builder = new StringBuilder(value.Length); + foreach (var ch in value) + { + if (char.IsLetterOrDigit(ch)) + { + builder.Append(char.ToLowerInvariant(ch)); + } + } + + return builder.ToString(); } } diff --git a/src/StellaOps.Feedser.Models/TASKS.md b/src/StellaOps.Feedser.Models/TASKS.md index 2ded64ca..8b0df934 100644 --- a/src/StellaOps.Feedser.Models/TASKS.md +++ b/src/StellaOps.Feedser.Models/TASKS.md @@ -12,7 +12,7 @@ |Range primitives for SemVer/EVR/NEVRA metadata|BE-Merge|Models|TODO – keep structured values without parsing logic; ensure merge/export parity.| |Provenance envelope field masks|BE-Merge|Models|TODO – guarantee traceability for each mapped field.| |Backward-compatibility playbook|BE-Merge, QA|Models|DONE – see `BACKWARD_COMPATIBILITY.md` for evolution policy/test checklist.| -|Golden canonical examples|QA|Models|DOING – `CanonicalExampleFactory` + tests added; fixtures need regeneration hookup to enable easy updates.| +|Golden canonical examples|QA|Models|DONE – added `/p:UpdateGoldens=true` test hook wiring `UPDATE_GOLDENS=1` so canonical fixtures regenerate via `dotnet test`; docs/tests unchanged.| |Serialization determinism regression tests|QA|Models|TODO – automate hash comparisons across locales/runs.| -|Severity normalization helpers|BE-Merge|Models|DOING – helper introduced to normalize vendor severities; refine synonym coverage.| -|AffectedPackage status glossary & guardrails|BE-Merge|Models|DOING – status catalog with validation added; monitor for additional vendor values.| +|Severity normalization helpers|BE-Merge|Models|DONE – helper now normalizes compound vendor labels/priority tiers with expanded synonym coverage and regression tests.| +|AffectedPackage status glossary & guardrails|BE-Merge|Models|DONE – catalog now exposes deterministic listing, TryNormalize helpers, and synonym coverage for vendor phrases (not vulnerable, workaround available, etc.).| diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json index 600f4fa1..711c20d1 100644 --- a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json +++ b/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json @@ -91,7 +91,7 @@ "url": "https://www.cve.org/CVERecord?id=CVE-2024-9991" } ], - "severity": "high vendor", + "severity": "high", "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).", "title": "Multiple vulnerabilities in Example Gateway" -} \ No newline at end of file +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs index c7bd6ae4..6e023e46 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs +++ b/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs @@ -22,6 +22,7 @@ public sealed class ExportStateManagerTests cursor: "cursor-1", targetRepository: "registry.local/json", exporterVersion: "1.0.0", + resetBaseline: true, cancellationToken: CancellationToken.None); Assert.Equal("export:json", record.Id); @@ -35,6 +36,93 @@ public sealed class ExportStateManagerTests Assert.Equal(timeProvider.Now, record.UpdatedAt); } + [Fact] + public async Task StoreFullExport_ResetBaselineOverridesExisting() + { + var store = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-20T12:00:00Z")); + var manager = new ExportStateManager(store, timeProvider); + + await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T120000Z", + exportDigest: "sha256:base", + cursor: "cursor-base", + targetRepository: null, + exporterVersion: "1.0.0", + resetBaseline: true, + cancellationToken: CancellationToken.None); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + var withoutReset = await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T120500Z", + exportDigest: "sha256:new", + cursor: "cursor-new", + targetRepository: null, + exporterVersion: "1.0.1", + resetBaseline: false, + cancellationToken: CancellationToken.None); + + Assert.Equal("20240720T120000Z", withoutReset.BaseExportId); + Assert.Equal("sha256:base", withoutReset.BaseDigest); + Assert.Equal("sha256:new", withoutReset.LastFullDigest); + Assert.Equal("cursor-new", withoutReset.ExportCursor); + Assert.Equal(timeProvider.Now, withoutReset.UpdatedAt); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + var reset = await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240720T121000Z", + exportDigest: "sha256:final", + cursor: "cursor-final", + targetRepository: null, + exporterVersion: "1.0.2", + resetBaseline: true, + cancellationToken: CancellationToken.None); + + Assert.Equal("20240720T121000Z", reset.BaseExportId); + Assert.Equal("sha256:final", reset.BaseDigest); + Assert.Equal("sha256:final", reset.LastFullDigest); + Assert.Null(reset.LastDeltaDigest); + Assert.Equal("cursor-final", reset.ExportCursor); + Assert.Equal(timeProvider.Now, reset.UpdatedAt); + } + + [Fact] + public async Task StoreFullExport_ResetsBaselineWhenRepositoryChanges() + { + var store = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-21T08:00:00Z")); + var manager = new ExportStateManager(store, timeProvider); + + await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240721T080000Z", + exportDigest: "sha256:base", + cursor: "cursor-base", + targetRepository: "registry/v1/json", + exporterVersion: "1.0.0", + resetBaseline: true, + cancellationToken: CancellationToken.None); + + timeProvider.Advance(TimeSpan.FromMinutes(10)); + var updated = await manager.StoreFullExportAsync( + exporterId: "export:json", + exportId: "20240721T081000Z", + exportDigest: "sha256:new", + cursor: "cursor-new", + targetRepository: "registry/v2/json", + exporterVersion: "1.1.0", + resetBaseline: false, + cancellationToken: CancellationToken.None); + + Assert.Equal("20240721T081000Z", updated.BaseExportId); + Assert.Equal("sha256:new", updated.BaseDigest); + Assert.Equal("sha256:new", updated.LastFullDigest); + Assert.Equal("registry/v2/json", updated.TargetRepository); + } + [Fact] public async Task StoreDeltaExportRequiresBaseline() { @@ -63,6 +151,7 @@ public sealed class ExportStateManagerTests cursor: "cursor-1", targetRepository: null, exporterVersion: "1.0.0", + resetBaseline: true, cancellationToken: CancellationToken.None); timeProvider.Advance(TimeSpan.FromMinutes(10)); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs index c1c09acf..f2bff595 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs +++ b/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs @@ -31,6 +31,7 @@ public sealed class ExportStateManager string? cursor, string? targetRepository, string exporterVersion, + bool resetBaseline, CancellationToken cancellationToken) { ArgumentException.ThrowIfNullOrEmpty(exporterId); @@ -39,36 +40,59 @@ public sealed class ExportStateManager ArgumentException.ThrowIfNullOrEmpty(exporterVersion); var existing = await _store.FindAsync(exporterId, cancellationToken).ConfigureAwait(false); - var repository = string.IsNullOrWhiteSpace(targetRepository) ? existing?.TargetRepository : targetRepository; var now = _timeProvider.GetUtcNow(); - var baseExportId = existing?.BaseExportId ?? exportId; - var baseDigest = existing?.BaseDigest ?? exportDigest; + if (existing is null) + { + var resolvedRepository = string.IsNullOrWhiteSpace(targetRepository) ? null : targetRepository; + return await _store.UpsertAsync( + new ExportStateRecord( + exporterId, + BaseExportId: exportId, + BaseDigest: exportDigest, + LastFullDigest: exportDigest, + LastDeltaDigest: null, + ExportCursor: cursor ?? exportDigest, + TargetRepository: resolvedRepository, + ExporterVersion: exporterVersion, + UpdatedAt: now), + cancellationToken).ConfigureAwait(false); + } - var record = existing is null - ? new ExportStateRecord( - exporterId, - baseExportId, - baseDigest, - exportDigest, - LastDeltaDigest: null, - ExportCursor: cursor ?? exportDigest, - TargetRepository: repository, - ExporterVersion: exporterVersion, - UpdatedAt: now) + var repositorySpecified = !string.IsNullOrWhiteSpace(targetRepository); + var resolvedRepo = repositorySpecified ? targetRepository : existing.TargetRepository; + var repositoryChanged = repositorySpecified + && !string.Equals(existing.TargetRepository, targetRepository, StringComparison.Ordinal); + + var shouldResetBaseline = + resetBaseline + || string.IsNullOrWhiteSpace(existing.BaseExportId) + || string.IsNullOrWhiteSpace(existing.BaseDigest) + || repositoryChanged; + + var updatedRecord = shouldResetBaseline + ? existing with + { + BaseExportId = exportId, + BaseDigest = exportDigest, + LastFullDigest = exportDigest, + LastDeltaDigest = null, + ExportCursor = cursor ?? exportDigest, + TargetRepository = resolvedRepo, + ExporterVersion = exporterVersion, + UpdatedAt = now, + } : existing with { - BaseExportId = baseExportId, - BaseDigest = baseDigest, LastFullDigest = exportDigest, LastDeltaDigest = null, ExportCursor = cursor ?? existing.ExportCursor, - TargetRepository = repository, + TargetRepository = resolvedRepo, ExporterVersion = exporterVersion, UpdatedAt = now, }; - return await _store.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + return await _store.UpsertAsync(updatedRecord, cancellationToken).ConfigureAwait(false); } public async Task StoreDeltaExportAsync(