feat: Add VEX compact fixture and implement offline verifier for Findings Ledger exports

- Introduced a new VEX compact fixture for testing purposes. - Implemented `verify_export.py` script to validate Findings Ledger exports, ensuring deterministic ordering and applying redaction manifests. - Added a lightweight stub `HarnessRunner` for unit tests to validate ledger hashing expectations. - Documented tasks related to the Mirror Creator. - Created models for entropy signals and implemented the `EntropyPenaltyCalculator` to compute penalties based on scanner outputs. - Developed unit tests for `EntropyPenaltyCalculator` to ensure correct penalty calculations and handling of edge cases. - Added tests for symbol ID normalization in the reachability scanner. - Enhanced console status service with comprehensive unit tests for connection handling and error recovery. - Included Cosign tool version 2.6.0 with checksums for various platforms.
2025-12-02 21:08:01 +02:00
parent 6d049905c7
commit 47168fec38
146 changed files with 4329 additions and 549 deletions
--- a/src/Concelier/__Analyzers/StellaOps.Concelier.Analyzers/AnalyzerReleases.Unshipped.md
+++ b/src/Concelier/__Analyzers/StellaOps.Concelier.Analyzers/AnalyzerReleases.Unshipped.md
@@ -2,4 +2,4 @@

 ### Unreleased

-No analyzer rules currently scheduled for release.
+- CONCELIER0004: Flag direct `new HttpClient()` usage inside `StellaOps.Concelier.Connector*` namespaces; require sandboxed `IHttpClientFactory` to enforce allow/deny lists.
--- a/src/Concelier/__Analyzers/StellaOps.Concelier.Analyzers/ConnectorHttpClientSandboxAnalyzer.cs
+++ b/src/Concelier/__Analyzers/StellaOps.Concelier.Analyzers/ConnectorHttpClientSandboxAnalyzer.cs
@@ -0,0 +1,53 @@
+using System.Collections.Immutable;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
+using Microsoft.CodeAnalysis.Diagnostics;
+
+namespace StellaOps.Concelier.Analyzers;
+
+[DiagnosticAnalyzer(LanguageNames.CSharp)]
+public sealed class ConnectorHttpClientSandboxAnalyzer : DiagnosticAnalyzer
+{
+    public const string DiagnosticId = "CONCELIER0004";
+
+    private static readonly DiagnosticDescriptor Rule = new(
+        id: DiagnosticId,
+        title: "Connector HTTP clients must use sandboxed factory",
+        messageFormat: "Use IHttpClientFactory or connector sandbox helpers instead of 'new HttpClient()' inside Concelier connectors.",
+        category: "Sandbox",
+        defaultSeverity: DiagnosticSeverity.Warning,
+        isEnabledByDefault: true,
+        description: "Direct HttpClient construction bypasses connector allowlist/denylist and proxy policies. Use IHttpClientFactory or sandboxed handlers.");
+
+    public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => ImmutableArray.Create(Rule);
+
+    public override void Initialize(AnalysisContext context)
+    {
+        context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None);
+        context.EnableConcurrentExecution();
+        context.RegisterSyntaxNodeAction(AnalyzeObjectCreation, SyntaxKind.ObjectCreationExpression);
+    }
+
+    private static void AnalyzeObjectCreation(SyntaxNodeAnalysisContext context)
+    {
+        if (context.Node is not ObjectCreationExpressionSyntax objectCreation)
+        {
+            return;
+        }
+
+        var type = context.SemanticModel.GetTypeInfo(objectCreation, context.CancellationToken).Type;
+        if (type?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat) != "global::System.Net.Http.HttpClient")
+        {
+            return;
+        }
+
+        var containingSymbol = context.ContainingSymbol?.ContainingNamespace?.ToDisplayString();
+        if (containingSymbol is null || !containingSymbol.StartsWith("StellaOps.Concelier.Connector"))
+        {
+            return;
+        }
+
+        context.ReportDiagnostic(Diagnostic.Create(Rule, objectCreation.GetLocation()));
+    }
+}
--- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Aoc/AdvisoryObservationWriteGuard.cs
+++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Aoc/AdvisoryObservationWriteGuard.cs
@@ -1,3 +1,5 @@
+using System;
+using System.Linq;
 using Microsoft.Extensions.Logging;
 using StellaOps.Concelier.Models.Observations;

@@ -29,6 +31,24 @@ public sealed class AdvisoryObservationWriteGuard : IAdvisoryObservationWriteGua
        ArgumentNullException.ThrowIfNull(observation);

        var newContentHash = observation.Upstream.ContentHash;
+        var signature = observation.Upstream.Signature;
+
+        if (!IsSha256(newContentHash))
+        {
+            _logger.LogWarning(
+                "Observation {ObservationId} rejected: content hash must be canonical sha256:<hex64> but was {ContentHash}",
+                observation.ObservationId,
+                newContentHash);
+            return ObservationWriteDisposition.RejectInvalidProvenance;
+        }
+
+        if (!SignatureShapeIsValid(signature))
+        {
+            _logger.LogWarning(
+                "Observation {ObservationId} rejected: signature metadata missing or inconsistent for provenance enforcement",
+                observation.ObservationId);
+            return ObservationWriteDisposition.RejectInvalidProvenance;
+        }

        if (string.IsNullOrWhiteSpace(existingContentHash))
        {
@@ -56,4 +76,36 @@ public sealed class AdvisoryObservationWriteGuard : IAdvisoryObservationWriteGua

        return ObservationWriteDisposition.RejectMutation;
    }
+
+    private static bool IsSha256(string? value)
+    {
+        if (string.IsNullOrWhiteSpace(value))
+        {
+            return false;
+        }
+
+        return value.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
+               && value.Length == "sha256:".Length + 64
+               && value["sha256:".Length..].All(c => Uri.IsHexDigit(c));
+    }
+
+    private static bool SignatureShapeIsValid(AdvisoryObservationSignature signature)
+    {
+        if (signature is null)
+        {
+            return false;
+        }
+
+        if (signature.Present)
+        {
+            return !string.IsNullOrWhiteSpace(signature.Format)
+                   && !string.IsNullOrWhiteSpace(signature.KeyId)
+                   && !string.IsNullOrWhiteSpace(signature.Signature);
+        }
+
+        // When signature is not present, auxiliary fields must be empty to prevent stale metadata.
+        return string.IsNullOrEmpty(signature.Format)
+               && string.IsNullOrEmpty(signature.KeyId)
+               && string.IsNullOrEmpty(signature.Signature);
+    }
 }
--- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Aoc/IAdvisoryObservationWriteGuard.cs
+++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Aoc/IAdvisoryObservationWriteGuard.cs
@@ -32,6 +32,11 @@ public enum ObservationWriteDisposition
    /// </summary>
    SkipIdentical,

+    /// <summary>
+    /// Observation is malformed (missing provenance/signature/hash guarantees) and must be rejected.
+    /// </summary>
+    RejectInvalidProvenance,
+
    /// <summary>
    /// Observation differs from existing - reject mutation (append-only violation).
    /// </summary>
--- a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Aoc/AdvisoryObservationWriteGuardTests.cs
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Aoc/AdvisoryObservationWriteGuardTests.cs
@@ -15,6 +15,10 @@ namespace StellaOps.Concelier.Core.Tests.Aoc;
 /// </summary>
 public sealed class AdvisoryObservationWriteGuardTests
 {
+    private const string HashA = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+    private const string HashB = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+    private const string HashC = "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";
+
    private readonly AdvisoryObservationWriteGuard _guard;

    public AdvisoryObservationWriteGuardTests()
@@ -26,7 +30,7 @@ public sealed class AdvisoryObservationWriteGuardTests
    public void ValidateWrite_NewObservation_ReturnsProceed()
    {
        // Arrange
-        var observation = CreateObservation("obs-1", "sha256:abc123");
+        var observation = CreateObservation("obs-1", HashA);

        // Act
        var result = _guard.ValidateWrite(observation, existingContentHash: null);
@@ -39,7 +43,7 @@ public sealed class AdvisoryObservationWriteGuardTests
    public void ValidateWrite_NewObservation_WithEmptyExistingHash_ReturnsProceed()
    {
        // Arrange
-        var observation = CreateObservation("obs-2", "sha256:def456");
+        var observation = CreateObservation("obs-2", HashB);

        // Act
        var result = _guard.ValidateWrite(observation, existingContentHash: "");
@@ -52,7 +56,7 @@ public sealed class AdvisoryObservationWriteGuardTests
    public void ValidateWrite_NewObservation_WithWhitespaceExistingHash_ReturnsProceed()
    {
        // Arrange
-        var observation = CreateObservation("obs-3", "sha256:ghi789");
+        var observation = CreateObservation("obs-3", HashC);

        // Act
        var result = _guard.ValidateWrite(observation, existingContentHash: "   ");
@@ -65,11 +69,10 @@ public sealed class AdvisoryObservationWriteGuardTests
    public void ValidateWrite_IdenticalContent_ReturnsSkipIdentical()
    {
        // Arrange
-        const string contentHash = "sha256:abc123";
-        var observation = CreateObservation("obs-4", contentHash);
+        var observation = CreateObservation("obs-4", HashA);

        // Act
-        var result = _guard.ValidateWrite(observation, existingContentHash: contentHash);
+        var result = _guard.ValidateWrite(observation, existingContentHash: HashA);

        // Assert
        result.Should().Be(ObservationWriteDisposition.SkipIdentical);
@@ -79,10 +82,10 @@ public sealed class AdvisoryObservationWriteGuardTests
    public void ValidateWrite_IdenticalContent_CaseInsensitive_ReturnsSkipIdentical()
    {
        // Arrange
-        var observation = CreateObservation("obs-5", "SHA256:ABC123");
+        var observation = CreateObservation("obs-5", "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");

        // Act
-        var result = _guard.ValidateWrite(observation, existingContentHash: "sha256:abc123");
+        var result = _guard.ValidateWrite(observation, existingContentHash: HashA);

        // Assert
        result.Should().Be(ObservationWriteDisposition.SkipIdentical);
@@ -92,10 +95,10 @@ public sealed class AdvisoryObservationWriteGuardTests
    public void ValidateWrite_DifferentContent_ReturnsRejectMutation()
    {
        // Arrange
-        var observation = CreateObservation("obs-6", "sha256:newcontent");
+        var observation = CreateObservation("obs-6", HashB);

        // Act
-        var result = _guard.ValidateWrite(observation, existingContentHash: "sha256:oldcontent");
+        var result = _guard.ValidateWrite(observation, existingContentHash: HashA);

        // Assert
        result.Should().Be(ObservationWriteDisposition.RejectMutation);
@@ -113,9 +116,8 @@ public sealed class AdvisoryObservationWriteGuardTests
    }

    [Theory]
-    [InlineData("sha256:a", "sha256:b")]
-    [InlineData("sha256:hash1", "sha256:hash2")]
-    [InlineData("md5:abc", "sha256:abc")]
+    [InlineData(HashB, HashC)]
+    [InlineData(HashC, HashA)]
    public void ValidateWrite_ContentMismatch_ReturnsRejectMutation(string newHash, string existingHash)
    {
        // Arrange
@@ -129,9 +131,8 @@ public sealed class AdvisoryObservationWriteGuardTests
    }

    [Theory]
-    [InlineData("sha256:identical")]
-    [InlineData("SHA256:IDENTICAL")]
-    [InlineData("sha512:longerhash1234567890")]
+    [InlineData(HashA)]
+    [InlineData("SHA256:BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB")]
    public void ValidateWrite_ExactMatch_ReturnsSkipIdentical(string hash)
    {
        // Arrange
@@ -144,7 +145,41 @@ public sealed class AdvisoryObservationWriteGuardTests
        result.Should().Be(ObservationWriteDisposition.SkipIdentical);
    }

-    private static AdvisoryObservation CreateObservation(string observationId, string contentHash)
+    [Theory]
+    [InlineData("md5:abc")]
+    [InlineData("sha256:short")]
+    public void ValidateWrite_InvalidHash_ReturnsRejectInvalidProvenance(string hash)
+    {
+        var observation = CreateObservation("obs-invalid-hash", hash);
+
+        var result = _guard.ValidateWrite(observation, existingContentHash: null);
+
+        result.Should().Be(ObservationWriteDisposition.RejectInvalidProvenance);
+    }
+
+    [Fact]
+    public void ValidateWrite_SignaturePresentMissingFields_ReturnsRejectInvalidProvenance()
+    {
+        var badSignature = new AdvisoryObservationSignature(true, null, null, null);
+        var observation = CreateObservation("obs-bad-sig", HashA, badSignature);
+
+        var result = _guard.ValidateWrite(observation, existingContentHash: null);
+
+        result.Should().Be(ObservationWriteDisposition.RejectInvalidProvenance);
+    }
+
+    [Fact]
+    public void Observation_TenantIsLowercased()
+    {
+        var observation = CreateObservation("obs-tenant", HashA, tenant: "Tenant:Mixed");
+        observation.Tenant.Should().Be("tenant:mixed");
+    }
+
+    private static AdvisoryObservation CreateObservation(
+        string observationId,
+        string contentHash,
+        AdvisoryObservationSignature? signatureOverride = null,
+        string tenant = "test-tenant")
    {
        var source = new AdvisoryObservationSource(
            vendor: "test-vendor",
@@ -152,11 +187,11 @@ public sealed class AdvisoryObservationWriteGuardTests
            api: "test-api",
            collectorVersion: "1.0.0");

-        var signature = new AdvisoryObservationSignature(
-            present: false,
-            format: null,
-            keyId: null,
-            signature: null);
+        var signature = signatureOverride ?? new AdvisoryObservationSignature(
+            present: true,
+            format: "dsse",
+            keyId: "test-key",
+            signature: "ZmFrZS1zaWc=");

        var upstream = new AdvisoryObservationUpstream(
            upstreamId: $"upstream-{observationId}",
@@ -184,7 +219,7 @@ public sealed class AdvisoryObservationWriteGuardTests

        return new AdvisoryObservation(
            observationId: observationId,
-            tenant: "test-tenant",
+            tenant: tenant,
            source: source,
            upstream: upstream,
            content: content,
--- a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/AdvisoryLinksetDeterminismTests.cs
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Linksets/AdvisoryLinksetDeterminismTests.cs
@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using FluentAssertions;
+using StellaOps.Concelier.Core.Linksets;
+using StellaOps.Concelier.Models;
+using Xunit;
+
+namespace StellaOps.Concelier.Core.Tests.Linksets;
+
+/// <summary>
+/// Determinism and provenance-focused tests aligned with CI1–CI10 gap remediation.
+/// </summary>
+public sealed class AdvisoryLinksetDeterminismTests
+{
+    [Fact]
+    public void IdempotencyKey_IsStableAcrossObservationOrdering()
+    {
+        // Arrange
+        var createdAt = new DateTimeOffset(2025, 12, 2, 0, 0, 0, TimeSpan.Zero);
+        var observationsA = ImmutableArray.Create("obs-b", "obs-a");
+        var observationsB = ImmutableArray.Create("obs-a", "obs-b");
+
+        var linksetA = new AdvisoryLinkset(
+            TenantId: "tenant-a",
+            Source: "nvd",
+            AdvisoryId: "CVE-2025-9999",
+            ObservationIds: observationsA,
+            Normalized: null,
+            Provenance: new AdvisoryLinksetProvenance(
+                ObservationHashes: new[] { "sha256:1111", "sha256:2222" },
+                ToolVersion: "1.0.0",
+                PolicyHash: "policy-hash-1"),
+            Confidence: 0.8,
+            Conflicts: null,
+            CreatedAt: createdAt,
+            BuiltByJobId: "job-1");
+
+        var linksetB = linksetA with { ObservationIds = observationsB };
+
+        // Act
+        var evtA = AdvisoryLinksetUpdatedEvent.FromLinkset(linksetA, null, "linkset-1", null);
+        var evtB = AdvisoryLinksetUpdatedEvent.FromLinkset(linksetB, null, "linkset-1", null);
+
+        // Assert
+        evtA.IdempotencyKey.Should().Be(evtB.IdempotencyKey);
+    }
+
+    [Fact]
+    public void Conflicts_AreDeterministicallyDedupedAndSourcesFilled()
+    {
+        // Arrange
+        var inputs = new[]
+        {
+            new LinksetCorrelation.Input(
+                Vendor: "nvd",
+                FetchedAt: DateTimeOffset.Parse("2025-12-01T00:00:00Z"),
+                Aliases: new[] { "CVE-2025-1111" },
+                Purls: Array.Empty<string>(),
+                Cpes: Array.Empty<string>(),
+                References: Array.Empty<string>()),
+            new LinksetCorrelation.Input(
+                Vendor: "vendor",
+                FetchedAt: DateTimeOffset.Parse("2025-12-01T00:05:00Z"),
+                Aliases: new[] { "CVE-2025-2222" },
+                Purls: Array.Empty<string>(),
+                Cpes: Array.Empty<string>(),
+                References: Array.Empty<string>())
+        };
+
+        var duplicateConflicts = new List<AdvisoryLinksetConflict>
+        {
+            new("aliases", "alias-inconsistency", new[] { "nvd:CVE-2025-1111", "vendor:CVE-2025-2222" }, null),
+            new("aliases", "alias-inconsistency", new[] { "nvd:CVE-2025-1111", "vendor:CVE-2025-2222" }, Array.Empty<string>())
+        };
+
+        // Act
+        var (_, conflicts) = LinksetCorrelation.Compute(inputs, duplicateConflicts);
+
+        // Assert
+        conflicts.Should().HaveCount(1);
+        conflicts[0].Field.Should().Be("aliases");
+        conflicts[0].Reason.Should().Be("alias-inconsistency");
+        conflicts[0].SourceIds.Should().ContainInOrder("nvd", "vendor");
+    }
+}
--- a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Schemas/SchemaManifestTests.cs
+++ b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Schemas/SchemaManifestTests.cs
@@ -0,0 +1,86 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text.Json;
+using FluentAssertions;
+using Xunit;
+
+namespace StellaOps.Concelier.Core.Tests.Schemas;
+
+/// <summary>
+/// Verifies schema bundle digests and offline bundle sample constraints (CI1–CI10).
+/// </summary>
+public sealed class SchemaManifestTests
+{
+    private static readonly string SchemaRoot = ResolveSchemaRoot();
+
+    [Fact]
+    public void SchemaManifest_DigestsMatchFilesystem()
+    {
+        var manifestPath = Path.Combine(SchemaRoot, "schema.manifest.json");
+        using var doc = JsonDocument.Parse(File.ReadAllText(manifestPath));
+
+        var files = doc.RootElement.GetProperty("files").EnumerateArray().ToArray();
+        files.Should().NotBeEmpty("schema manifest must contain at least one entry");
+
+        foreach (var fileEl in files)
+        {
+            var path = fileEl.GetProperty("path").GetString()!;
+            var expected = fileEl.GetProperty("sha256").GetString()!;
+
+            var fullPath = Path.Combine(SchemaRoot, path);
+            File.Exists(fullPath).Should().BeTrue($"manifest entry {path} should exist");
+
+            var actual = ComputeSha256(fullPath);
+            actual.Should().Be(expected, $"digest for {path} should be canonical");
+        }
+    }
+
+    [Fact]
+    public void OfflineBundleSample_RespectsStalenessAndHashes()
+    {
+        var samplePath = Path.Combine(SchemaRoot, "samples/offline-advisory-bundle.sample.json");
+        using var doc = JsonDocument.Parse(File.ReadAllText(samplePath));
+
+        var snapshot = doc.RootElement.GetProperty("snapshot");
+        var staleness = snapshot.GetProperty("stalenessHours").GetInt32();
+        staleness.Should().BeLessOrEqualTo(168, "offline bundles must cap snapshot staleness to 7 days");
+
+        var manifest = doc.RootElement.GetProperty("manifest").EnumerateArray().ToArray();
+        manifest.Should().NotBeEmpty();
+        foreach (var entry in manifest)
+        {
+            var hash = entry.GetProperty("sha256").GetString()!;
+            hash.Length.Should().Be(64);
+        }
+
+        var hashes = doc.RootElement.GetProperty("hashes");
+        hashes.GetProperty("sha256").GetString()!.Length.Should().Be(64);
+    }
+
+    private static string ComputeSha256(string path)
+    {
+        using var sha = SHA256.Create();
+        using var stream = File.OpenRead(path);
+        var hash = sha.ComputeHash(stream);
+        return Convert.ToHexString(hash).ToLowerInvariant();
+    }
+
+    private static string ResolveSchemaRoot()
+    {
+        var current = AppContext.BaseDirectory;
+        while (!string.IsNullOrEmpty(current))
+        {
+            var candidate = Path.Combine(current, "docs/modules/concelier/schemas");
+            if (Directory.Exists(candidate))
+            {
+                return candidate;
+            }
+
+            current = Directory.GetParent(current)?.FullName;
+        }
+
+        throw new DirectoryNotFoundException("Unable to locate docs/modules/concelier/schemas from test base directory.");
+    }
+}