feat: Add VEX Lens CI and Load Testing Plan

- Introduced a comprehensive CI job structure for VEX Lens, including build, test, linting, and load testing. - Defined load test parameters and SLOs for VEX Lens API and Issuer Directory. - Created Grafana dashboards and alerting mechanisms for monitoring API performance and error rates. - Established offline posture guidelines for CI jobs and load testing. feat: Implement deterministic projection verification script - Added `verify_projection.sh` script for verifying the integrity of projection exports against expected hashes. - Ensured robust error handling for missing files and hash mismatches. feat: Develop Vuln Explorer CI and Ops Plan - Created CI jobs for Vuln Explorer, including build, test, and replay verification. - Implemented backup and disaster recovery strategies for MongoDB and Redis. - Established Merkle anchoring verification and automation for ledger projector. feat: Introduce EventEnvelopeHasher for hashing event envelopes - Implemented `EventEnvelopeHasher` to compute SHA256 hashes for event envelopes. feat: Add Risk Store and Dashboard components - Developed `RiskStore` for managing risk data and state. - Created `RiskDashboardComponent` for displaying risk profiles with filtering capabilities. - Implemented unit tests for `RiskStore` and `RiskDashboardComponent`. feat: Enhance Vulnerability Detail Component - Developed `VulnerabilityDetailComponent` for displaying detailed information about vulnerabilities. - Implemented error handling for missing vulnerability IDs and loading failures.
2025-12-02 07:18:28 +02:00
parent 44171930ff
commit 885ce86af4
83 changed files with 2090 additions and 97 deletions
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/IDsseEnvelopeSigner.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/IDsseEnvelopeSigner.cs
@@ -0,0 +1,57 @@
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Worker.Processing.Surface;
+
+internal sealed record DsseEnvelope(string MediaType, string Uri, string Digest, ReadOnlyMemory<byte> Content);
+
+internal interface IDsseEnvelopeSigner
+{
+    Task<DsseEnvelope> SignAsync(string payloadType, ReadOnlyMemory<byte> content, string suggestedKind, string merkleRoot, string? view, CancellationToken cancellationToken);
+}
+
+/// <summary>
+/// Deterministic fallback signer that encodes sha256 hash as the signature. Replace with real Attestor/Signer when available.
+/// </summary>
+internal sealed class DeterministicDsseEnvelopeSigner : IDsseEnvelopeSigner
+{
+    public Task<DsseEnvelope> SignAsync(string payloadType, ReadOnlyMemory<byte> content, string suggestedKind, string merkleRoot, string? view, CancellationToken cancellationToken)
+    {
+        var signature = ComputeSha256Hex(content.Span);
+        var envelope = new
+        {
+            payloadType,
+            payload = Base64UrlEncode(content.Span),
+            signatures = new[]
+            {
+                new { keyid = "scanner-deterministic", sig = Base64UrlEncode(Encoding.UTF8.GetBytes(signature)) }
+            }
+        };
+
+        var json = JsonSerializer.Serialize(envelope, new JsonSerializerOptions(JsonSerializerDefaults.Web)
+        {
+            WriteIndented = false
+        });
+
+        var bytes = Encoding.UTF8.GetBytes(json);
+        var digest = $"sha256:{signature}";
+        var uri = $"cas://attestations/{suggestedKind}/{signature}.json";
+
+        return Task.FromResult(new DsseEnvelope("application/vnd.dsse+json", uri, digest, bytes));
+    }
+
+    private static string ComputeSha256Hex(ReadOnlySpan<byte> data)
+    {
+        Span<byte> hash = stackalloc byte[32];
+        System.Security.Cryptography.SHA256.HashData(data, hash);
+        return Convert.ToHexString(hash).ToLowerInvariant();
+    }
+
+    private static string Base64UrlEncode(ReadOnlySpan<byte> data)
+    {
+        var base64 = Convert.ToBase64String(data);
+        return base64.Replace("+", "-").Replace("/", "_").TrimEnd('=');
+    }
+}
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestPublisher.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestPublisher.cs
@@ -117,7 +117,7 @@ internal sealed class SurfaceManifestPublisher : ISurfaceManifestPublisher
                WorkerInstance = request.WorkerInstance,
                Attempt = request.Attempt
            },
-            Artifacts = artifacts.ToImmutableArray(),
+            Artifacts = AttachAttestations(artifacts).ToImmutableArray(),
            DeterminismMerkleRoot = request.DeterminismMerkleRoot,
            ReplayBundle = string.IsNullOrWhiteSpace(request.ReplayBundleUri)
                ? null
@@ -196,6 +196,61 @@ internal sealed class SurfaceManifestPublisher : ISurfaceManifestPublisher
            DeterminismMerkleRoot: request.DeterminismMerkleRoot);
    }

+    private static IReadOnlyList<SurfaceManifestArtifact> AttachAttestations(IReadOnlyList<SurfaceManifestArtifact> artifacts)
+    {
+        if (artifacts.Count == 0)
+        {
+            return artifacts;
+        }
+
+        var dsseArtifacts = artifacts.Where(a => a.Kind.EndsWith(".dsse", StringComparison.Ordinal)).ToList();
+        if (dsseArtifacts.Count == 0)
+        {
+            return artifacts;
+        }
+
+        var updated = artifacts.ToList();
+
+        foreach (var dsse in dsseArtifacts)
+        {
+            var targetKind = dsse.Kind switch
+            {
+                "composition.recipe.dsse" => "composition.recipe",
+                "layer.fragments.dsse" => "layer.fragments",
+                _ => null
+            };
+
+            if (targetKind is null)
+            {
+                continue;
+            }
+
+            var targetIndex = updated.FindIndex(a => string.Equals(a.Kind, targetKind, StringComparison.Ordinal));
+            if (targetIndex < 0)
+            {
+                continue;
+            }
+
+            var attestation = new SurfaceManifestAttestation
+            {
+                Kind = "dsse",
+                MediaType = dsse.MediaType,
+                Digest = dsse.Digest,
+                Uri = dsse.Uri
+            };
+
+            var existing = updated[targetIndex].Attestations ?? Array.Empty<SurfaceManifestAttestation>();
+            var attList = existing.Concat(new[] { attestation })
+                .OrderBy(a => a.Kind, StringComparer.Ordinal)
+                .ThenBy(a => a.Uri, StringComparer.Ordinal)
+                .ToList();
+
+            updated[targetIndex] = updated[targetIndex] with { Attestations = attList };
+        }
+
+        return updated;
+    }
+
    private async Task<SurfaceManifestArtifact> StorePayloadAsync(SurfaceManifestPayload payload, string tenant, CancellationToken cancellationToken)
    {
        var digest = ComputeDigest(payload.Content.Span);
--- a/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs
@@ -44,6 +44,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
    private readonly ICryptoHash _hash;
    private readonly IRubyPackageInventoryStore _rubyPackageStore;
    private readonly Determinism.DeterminismContext _determinism;
+    private readonly IDsseEnvelopeSigner _dsseSigner;
    private readonly string _componentVersion;

    public SurfaceManifestStageExecutor(
@@ -55,7 +56,8 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
        ILogger<SurfaceManifestStageExecutor> logger,
        ICryptoHash hash,
        IRubyPackageInventoryStore rubyPackageStore,
-        Determinism.DeterminismContext determinism)
+        Determinism.DeterminismContext determinism,
+        IDsseEnvelopeSigner dsseSigner)
    {
        _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
        _manifestWriter = manifestWriter ?? throw new ArgumentNullException(nameof(manifestWriter));
@@ -66,6 +68,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
        _hash = hash ?? throw new ArgumentNullException(nameof(hash));
        _rubyPackageStore = rubyPackageStore ?? throw new ArgumentNullException(nameof(rubyPackageStore));
        _determinism = determinism ?? throw new ArgumentNullException(nameof(determinism));
+        _dsseSigner = dsseSigner ?? throw new ArgumentNullException(nameof(dsseSigner));
        _componentVersion = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
    }

@@ -78,10 +81,10 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
        var payloads = CollectPayloads(context);
        await PersistRubyPackagesAsync(context, cancellationToken).ConfigureAwait(false);

-        var determinismPayload = BuildDeterminismPayload(context, payloads, out var merkleRoot);
-        if (determinismPayload is not null)
+        var determinismPayloads = BuildDeterminismPayloads(context, payloads, out var merkleRoot);
+        if (determinismPayloads is not null && determinismPayloads.Count > 0)
        {
-            payloads.Add(determinismPayload);
+            payloads.AddRange(determinismPayloads);
        }
        if (payloads.Count == 0)
        {
@@ -251,7 +254,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
        return payloads;
    }

-    private SurfaceManifestPayload? BuildDeterminismPayload(ScanJobContext context, IEnumerable<SurfaceManifestPayload> payloads, out string? merkleRoot)
+    private IReadOnlyList<SurfaceManifestPayload> BuildDeterminismPayloads(ScanJobContext context, IEnumerable<SurfaceManifestPayload> payloads, out string? merkleRoot)
    {
        merkleRoot = null;
        var pins = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
@@ -283,9 +286,10 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
        var evidence = new Determinism.DeterminismEvidence(artifactHashes, recipeSha256);
        context.Analysis.Set(ScanAnalysisKeys.DeterminismEvidence, evidence);

+        var payloadList = payloads.ToList();
+
        // Publish composition recipe as a manifest artifact for offline replay.
-        payloads = payloads.ToList();
-        ((List<SurfaceManifestPayload>)payloads).Add(new SurfaceManifestPayload(
+        payloadList.Add(new SurfaceManifestPayload(
            ArtifactDocumentType.CompositionRecipe,
            ArtifactDocumentFormat.CompositionRecipeJson,
            Kind: "composition.recipe",
@@ -297,14 +301,61 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
                ["merkleRoot"] = recipeSha256,
            }));

+        // Attach DSSE envelope for the recipe (deterministic local signature = sha256 hash bytes).
+        var recipeDsse = _dsseSigner.SignAsync(
+            payloadType: "application/vnd.stellaops.composition.recipe+json",
+            content: recipeBytes,
+            suggestedKind: "composition.recipe.dsse",
+            merkleRoot: recipeSha256,
+            view: null,
+            cancellationToken: CancellationToken.None).Result;
+        payloadList.Add(new SurfaceManifestPayload(
+            ArtifactDocumentType.Attestation,
+            ArtifactDocumentFormat.DsseJson,
+            Kind: "composition.recipe.dsse",
+            MediaType: recipeDsse.MediaType,
+            Content: recipeDsse.Content,
+            Metadata: new Dictionary<string, string>
+            {
+                ["merkleRoot"] = recipeSha256,
+                ["payloadType"] = "application/vnd.dsse+json"
+            }));
+
+        // Attach DSSE envelope for layer fragments when present.
+        foreach (var fragmentPayload in payloadList.Where(p => p.Kind == "layer.fragments"))
+        {
+            var dsse = _dsseSigner.SignAsync(
+                payloadType: fragmentPayload.MediaType,
+                content: fragmentPayload.Content,
+                suggestedKind: "layer.fragments.dsse",
+                merkleRoot: recipeSha256,
+                view: fragmentPayload.View,
+                cancellationToken: CancellationToken.None).Result;
+
+            payloadList.Add(new SurfaceManifestPayload(
+                ArtifactDocumentType.Attestation,
+                ArtifactDocumentFormat.DsseJson,
+                Kind: "layer.fragments.dsse",
+                MediaType: dsse.MediaType,
+                Content: dsse.Content,
+                View: fragmentPayload.View,
+                Metadata: new Dictionary<string, string>
+                {
+                    ["merkleRoot"] = recipeSha256,
+                    ["payloadType"] = fragmentPayload.MediaType
+                }));
+        }
+
        var json = JsonSerializer.Serialize(report, JsonOptions);
-        return new SurfaceManifestPayload(
+        payloadList.Add(new SurfaceManifestPayload(
            ArtifactDocumentType.SurfaceObservation,
            ArtifactDocumentFormat.ObservationJson,
            Kind: "determinism.json",
            MediaType: "application/json",
            Content: Encoding.UTF8.GetBytes(json),
-            View: "replay");
+            View: "replay"));
+
+        return payloadList.Skip(payloads.Count()).ToList();
    }

    private static (Dictionary<string, string> Hashes, byte[] RecipeBytes, string RecipeSha256) BuildCompositionRecipe(IEnumerable<SurfaceManifestPayload> payloads)
@@ -332,6 +383,48 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
        return (new Dictionary<string, string>(map, StringComparer.OrdinalIgnoreCase), recipeBytes, merkleRoot);
    }

+    private SurfaceManifestPayload BuildDsseEnvelopePayload(
+        string payloadType,
+        ReadOnlyMemory<byte> content,
+        string kind,
+        string mediaType,
+        string merkleRoot)
+    {
+        var signature = ComputeDigest(content.Span).Replace("sha256:", string.Empty, StringComparison.OrdinalIgnoreCase);
+        var envelope = new
+        {
+            payloadType,
+            payload = Base64UrlEncode(content.Span),
+            signatures = new[]
+            {
+                new
+                {
+                    keyid = "scanner-offline",
+                    sig = Base64UrlEncode(Encoding.UTF8.GetBytes(signature))
+                }
+            }
+        };
+
+        var json = JsonSerializer.Serialize(envelope, JsonOptions);
+        return new SurfaceManifestPayload(
+            ArtifactDocumentType.Attestation,
+            ArtifactDocumentFormat.DsseJson,
+            Kind: kind,
+            MediaType: mediaType,
+            Content: Encoding.UTF8.GetBytes(json),
+            Metadata: new Dictionary<string, string>
+            {
+                ["merkleRoot"] = merkleRoot,
+                ["payloadType"] = payloadType
+            });
+    }
+
+    private static string Base64UrlEncode(ReadOnlySpan<byte> data)
+    {
+        var base64 = Convert.ToBase64String(data);
+        return base64.Replace("+", "-").Replace("/", "_").TrimEnd('=');
+    }
+
    private static string? GetReplayBundleUri(ScanJobContext context)
        => context.Lease.Metadata.TryGetValue("replay.bundle.uri", out var value) && !string.IsNullOrWhiteSpace(value)
            ? value.Trim()
--- a/src/Scanner/StellaOps.Scanner.Worker/Program.cs
+++ b/src/Scanner/StellaOps.Scanner.Worker/Program.cs
@@ -99,8 +99,9 @@ if (!string.IsNullOrWhiteSpace(connectionString))
 {
    builder.Services.AddScannerStorage(storageSection);
    builder.Services.AddSingleton<IConfigureOptions<ScannerStorageOptions>, ScannerStorageSurfaceSecretConfigurator>();
-    builder.Services.AddSingleton<ISurfaceManifestPublisher, SurfaceManifestPublisher>();
-    builder.Services.AddSingleton<IScanStageExecutor, SurfaceManifestStageExecutor>();
+builder.Services.AddSingleton<ISurfaceManifestPublisher, SurfaceManifestPublisher>();
+builder.Services.AddSingleton<IScanStageExecutor, SurfaceManifestStageExecutor>();
+builder.Services.AddSingleton<IDsseEnvelopeSigner, DeterministicDsseEnvelopeSigner>();
 }
 else
 {