up

src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/Determinism/DeterminismHarness.cs

@@ -0,0 +1,127 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace StellaOps.Scanner.Worker.Tests.Determinism;
+
+/// <summary>
+/// Lightweight determinism harness used in tests to score repeated scanner runs.
+/// Groups runs by image digest, compares artefact hashes to the baseline (run index 0),
+/// and produces a report compatible with determinism.json expectations.
+/// </summary>
+internal static class DeterminismHarness
+{
+    public static DeterminismReport Compute(IEnumerable<DeterminismRunInput> runs, double imageThreshold = 0.90, double overallThreshold = 0.95)
+    {
+        ArgumentNullException.ThrowIfNull(runs);
+
+        var grouped = runs
+            .GroupBy(r => r.ImageDigest, StringComparer.OrdinalIgnoreCase)
+            .ToDictionary(g => g.Key, g => g.OrderBy(r => r.RunIndex).ToList(), StringComparer.OrdinalIgnoreCase);
+
+        var imageReports = new List<DeterminismImageReport>();
+        var totalRuns = 0;
+        var totalIdentical = 0;
+
+        foreach (var (image, entries) in grouped)
+        {
+            if (entries.Count == 0)
+            {
+                continue;
+            }
+
+            var baseline = entries[0];
+            var baselineHashes = HashArtifacts(baseline.Artifacts);
+            var runReports = new List<DeterminismRunReport>();
+            var identical = 0;
+
+            foreach (var run in entries)
+            {
+                var hashes = HashArtifacts(run.Artifacts);
+                var diff = hashes
+                    .Where(kv => !baselineHashes.TryGetValue(kv.Key, out var baselineHash) || !string.Equals(baselineHash, kv.Value, StringComparison.Ordinal))
+                    .Select(kv => kv.Key)
+                    .OrderBy(k => k, StringComparer.Ordinal)
+                    .ToArray();
+
+                var isIdentical = diff.Length == 0;
+                if (isIdentical)
+                {
+                    identical++;
+                }
+
+                runReports.Add(new DeterminismRunReport(run.RunIndex, hashes, diff));
+            }
+
+            var score = entries.Count == 0 ? 0d : (double)identical / entries.Count;
+            imageReports.Add(new DeterminismImageReport(image, entries.Count, identical, score, baselineHashes, runReports));
+
+            totalRuns += entries.Count;
+            totalIdentical += identical;
+        }
+
+        var overallScore = totalRuns == 0 ? 0d : (double)totalIdentical / totalRuns;
+
+        return new DeterminismReport(
+            OverallScore: overallScore,
+            OverallThreshold: overallThreshold,
+            ImageThreshold: imageThreshold,
+            Images: imageReports.OrderBy(r => r.ImageDigest, StringComparer.Ordinal).ToList());
+    }
+
+    private static IReadOnlyDictionary<string, string> HashArtifacts(IReadOnlyDictionary<string, string> artifacts)
+    {
+        var map = new Dictionary<string, string>(StringComparer.Ordinal);
+        foreach (var kv in artifacts)
+        {
+            var digest = Sha256Hex(kv.Value);
+            map[kv.Key] = digest;
+        }
+
+        return map;
+    }
+
+    private static string Sha256Hex(string content)
+    {
+        using var sha = SHA256.Create();
+        var bytes = Encoding.UTF8.GetBytes(content ?? string.Empty);
+        var hash = sha.ComputeHash(bytes);
+        return Convert.ToHexString(hash).ToLowerInvariant();
+    }
+}
+
+internal sealed record DeterminismRunInput(string ImageDigest, int RunIndex, IReadOnlyDictionary<string, string> Artifacts);
+
+internal sealed record DeterminismReport(
+    double OverallScore,
+    double OverallThreshold,
+    double ImageThreshold,
+    IReadOnlyList<DeterminismImageReport> Images)
+{
+    public string ToJson()
+    {
+        var options = new JsonSerializerOptions(JsonSerializerDefaults.Web)
+        {
+            WriteIndented = false,
+            DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
+        };
+        return JsonSerializer.Serialize(this, options);
+    }
+}
+
+internal sealed record DeterminismImageReport(
+    string ImageDigest,
+    int Runs,
+    int Identical,
+    double Score,
+    IReadOnlyDictionary<string, string> BaselineHashes,
+    IReadOnlyList<DeterminismRunReport> RunReports);
+
+internal sealed record DeterminismRunReport(
+    int RunIndex,
+    IReadOnlyDictionary<string, string> ArtifactHashes,
+    IReadOnlyList<string> NonDeterministicArtifacts);

src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/Determinism/DeterminismHarnessTests.cs

@@ -0,0 +1,44 @@
+using System.Collections.Generic;
+using System.Linq;
+using StellaOps.Scanner.Worker.Tests.Determinism;
+using Xunit;
+
+namespace StellaOps.Scanner.Worker.Tests.DeterminismTests;
+
+public sealed class DeterminismHarnessTests
+{
+    [Fact]
+    public void ComputeScores_FlagsDivergentArtifacts()
+    {
+        var runs = new[]
+        {
+            new DeterminismRunInput("sha256:image", 0, new Dictionary<string, string>
+            {
+                ["sbom.json"] = "sbom-a",
+                ["findings.ndjson"] = "findings-a",
+                ["log.ndjson"] = "log-1"
+            }),
+            new DeterminismRunInput("sha256:image", 1, new Dictionary<string, string>
+            {
+                ["sbom.json"] = "sbom-a",
+                ["findings.ndjson"] = "findings-a",
+                ["log.ndjson"] = "log-1"
+            }),
+            new DeterminismRunInput("sha256:image", 2, new Dictionary<string, string>
+            {
+                ["sbom.json"] = "sbom-a",
+                ["findings.ndjson"] = "findings-a",
+                ["log.ndjson"] = "log-2" // divergent
+            })
+        };
+
+        var report = DeterminismHarness.Compute(runs);
+
+        Assert.Equal(1.0 * 2 / 3, report.Images.Single().Score, precision: 3);
+        Assert.Equal(2, report.Images.Single().Identical);
+
+        var divergent = report.Images.Single().RunReports.Single(r => r.RunIndex == 2);
+        Assert.Contains("log.ndjson", divergent.NonDeterministicArtifacts);
+        Assert.DoesNotContain("sbom.json", divergent.NonDeterministicArtifacts);
+    }
+}

src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/Replay/ReplaySealedBundleStageExecutorTests.cs

@@ -0,0 +1,70 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging.Abstractions;
+using StellaOps.Scanner.Core.Contracts;
+using StellaOps.Scanner.Worker.Processing.Replay;
+using Xunit;
+
+namespace StellaOps.Scanner.Worker.Tests.Replay;
+
+public sealed class ReplaySealedBundleStageExecutorTests
+{
+    [Fact]
+    public async Task ExecuteAsync_SetsMetadata_WhenUriAndHashProvided()
+    {
+        var executor = new ReplaySealedBundleStageExecutor(NullLogger<ReplaySealedBundleStageExecutor>.Instance);
+        var context = TestContexts.Create();
+        context.Lease.Metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
+        context.Lease.Metadata["replay.bundle.sha256"] = "abc123";
+        context.Lease.Metadata["determinism.policy"] = "rev-1";
+        context.Lease.Metadata["determinism.feed"] = "feed-2";
+
+        await executor.ExecuteAsync(context, CancellationToken.None);
+
+        Assert.True(context.Analysis.TryGet<ReplaySealedBundleMetadata>(ScanAnalysisKeys.ReplaySealedBundleMetadata, out var metadata));
+        Assert.Equal("abc123", metadata.ManifestHash);
+        Assert.Equal("cas://replay/input.tar.zst", metadata.BundleUri);
+        Assert.Equal("rev-1", metadata.PolicySnapshotId);
+        Assert.Equal("feed-2", metadata.FeedSnapshotId);
+    }
+
+    [Fact]
+    public async Task ExecuteAsync_Skips_WhenHashMissing()
+    {
+        var executor = new ReplaySealedBundleStageExecutor(NullLogger<ReplaySealedBundleStageExecutor>.Instance);
+        var context = TestContexts.Create();
+        context.Lease.Metadata["replay.bundle.uri"] = "cas://replay/input.tar.zst";
+
+        await executor.ExecuteAsync(context, CancellationToken.None);
+
+        Assert.False(context.Analysis.TryGet<ReplaySealedBundleMetadata>(ScanAnalysisKeys.ReplaySealedBundleMetadata, out _));
+    }
+}
+
+internal static class TestContexts
+{
+    public static ScanJobContext Create()
+    {
+        var lease = new TestScanJobLease();
+        return new ScanJobContext(lease, TimeProvider.System, TimeProvider.System.GetUtcNow(), CancellationToken.None);
+    }
+
+    private sealed class TestScanJobLease : IScanJobLease
+    {
+        public string JobId => "job-1";
+        public string ScanId => "scan-1";
+        public int Attempt => 1;
+        public DateTimeOffset EnqueuedAtUtc => DateTimeOffset.UtcNow;
+        public DateTimeOffset LeasedAtUtc => DateTimeOffset.UtcNow;
+        public TimeSpan LeaseDuration => TimeSpan.FromMinutes(5);
+        public Dictionary<string, string> MutableMetadata { get; } = new();
+        public IReadOnlyDictionary<string, string> Metadata => MutableMetadata;
+
+        public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
+        public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
+        public ValueTask DisposeAsync() => ValueTask.CompletedTask;
+        public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask;
+        public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
+    }
+}