Add inline DSSE provenance documentation and Mongo schema

- Introduced a new document outlining the inline DSSE provenance for SBOM, VEX, scan, and derived events. - Defined the Mongo schema for event patches, including key fields for provenance and trust verification. - Documented the write path for ingesting provenance metadata and backfilling historical events. - Created CI/CD snippets for uploading DSSE attestations and generating provenance metadata. - Established Mongo indexes for efficient provenance queries and provided query recipes for various use cases. - Outlined policy gates for managing VEX decisions based on provenance verification. - Included UI nudges for displaying provenance information and implementation tasks for future enhancements. --- Implement reachability lattice and scoring model - Developed a comprehensive document detailing the reachability lattice and scoring model. - Defined core types for reachability states, evidence, and mitigations with corresponding C# models. - Established a scoring policy with base score contributions from various evidence classes. - Mapped reachability states to VEX gates and provided a clear overview of evidence sources. - Documented the event graph schema for persisting reachability data in MongoDB. - Outlined the integration of runtime probes for evidence collection and defined a roadmap for future tasks. --- Introduce uncertainty states and entropy scoring - Created a draft document for tracking uncertainty states and their impact on risk scoring. - Defined core uncertainty states with associated entropy values and evidence requirements. - Established a schema for storing uncertainty states alongside findings. - Documented the risk score calculation incorporating uncertainty and its effect on final risk assessments. - Provided policy guidelines for handling uncertainty in decision-making processes. - Outlined UI guidelines for displaying uncertainty information and suggested remediation actions. --- Add Ruby package inventory management - Implemented Ruby package inventory management with corresponding data models and storage mechanisms. - Created C# records for Ruby package inventory, artifacts, provenance, and runtime details. - Developed a repository for managing Ruby package inventory documents in MongoDB. - Implemented a service for storing and retrieving Ruby package inventories. - Added unit tests for the Ruby package inventory store to ensure functionality and data integrity.
2025-11-13 00:20:33 +02:00
parent 86be324fc0
commit 7040984215
41 changed files with 1955 additions and 76 deletions
--- a/src/Scanner/__Tests/StellaOps.Scanner.Storage.Tests/RubyPackageInventoryStoreTests.cs
+++ b/src/Scanner/__Tests/StellaOps.Scanner.Storage.Tests/RubyPackageInventoryStoreTests.cs
@@ -0,0 +1,121 @@
+using System.Collections.Generic;
+using Microsoft.Extensions.Options;
+using MongoDB.Driver;
+using StellaOps.Scanner.Core.Contracts;
+using StellaOps.Scanner.Storage;
+using StellaOps.Scanner.Storage.Mongo;
+using StellaOps.Scanner.Storage.Repositories;
+using StellaOps.Scanner.Storage.Services;
+using StellaOps.Scanner.Storage.Catalog;
+using Xunit;
+
+namespace StellaOps.Scanner.Storage.Tests;
+
+public sealed class RubyPackageInventoryStoreTests : IClassFixture<ScannerMongoFixture>
+{
+    private readonly ScannerMongoFixture _fixture;
+
+    public RubyPackageInventoryStoreTests(ScannerMongoFixture fixture)
+    {
+        _fixture = fixture;
+    }
+
+    [Fact]
+    public async Task StoreAsync_ThrowsWhenInventoryNull()
+    {
+        var store = CreateStore();
+        await Assert.ThrowsAsync<ArgumentNullException>(async () =>
+        {
+            RubyPackageInventory? inventory = null;
+            await store.StoreAsync(inventory!, CancellationToken.None);
+        });
+    }
+
+    [Fact]
+    public async Task GetAsync_ReturnsNullWhenMissing()
+    {
+        await ClearCollectionAsync();
+        var store = CreateStore();
+
+        var inventory = await store.GetAsync("scan-missing", CancellationToken.None);
+
+        Assert.Null(inventory);
+    }
+
+    [Fact]
+    public async Task StoreAsync_RoundTripsInventory()
+    {
+        await ClearCollectionAsync();
+        var store = CreateStore();
+
+        var scanId = $"scan-{Guid.NewGuid():n}";
+        var generatedAt = new DateTimeOffset(2025, 11, 12, 16, 10, 0, TimeSpan.Zero);
+
+        var packages = new[]
+        {
+            new RubyPackageArtifact(
+                Id: "purl::pkg:gem/rack@3.1.2",
+                Name: "rack",
+                Version: "3.1.2",
+                Source: "rubygems",
+                Platform: "ruby",
+                Groups: new[] {"default"},
+                DeclaredOnly: true,
+                RuntimeUsed: true,
+                Provenance: new RubyPackageProvenance("rubygems", "Gemfile.lock", "Gemfile.lock"),
+                Runtime: new RubyPackageRuntime(
+                    new[] { "config.ru" },
+                    new[] { "config.ru" },
+                    new[] { "require-static" }),
+                Metadata: new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["source"] = "rubygems",
+                    ["lockfile"] = "Gemfile.lock",
+                    ["groups"] = "default"
+                })
+        };
+
+        var inventory = new RubyPackageInventory(scanId, "sha256:image", generatedAt, packages);
+
+        await store.StoreAsync(inventory, CancellationToken.None);
+
+        var stored = await store.GetAsync(scanId, CancellationToken.None);
+
+        Assert.NotNull(stored);
+        Assert.Equal(scanId, stored!.ScanId);
+        Assert.Equal("sha256:image", stored.ImageDigest);
+        Assert.Equal(generatedAt, stored.GeneratedAtUtc);
+        Assert.Single(stored.Packages);
+        Assert.Equal("rack", stored.Packages[0].Name);
+        Assert.Equal("rubygems", stored.Packages[0].Source);
+    }
+
+    private async Task ClearCollectionAsync()
+    {
+        var provider = CreateProvider();
+        await provider.RubyPackages.DeleteManyAsync(Builders<RubyPackageInventoryDocument>.Filter.Empty);
+    }
+
+    private RubyPackageInventoryStore CreateStore()
+    {
+        var provider = CreateProvider();
+        var repository = new RubyPackageInventoryRepository(provider);
+        return new RubyPackageInventoryStore(repository);
+    }
+
+    private MongoCollectionProvider CreateProvider()
+    {
+        var options = Options.Create(new ScannerStorageOptions
+        {
+            Mongo = new MongoOptions
+            {
+                ConnectionString = _fixture.Runner.ConnectionString,
+                DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName,
+                UseMajorityReadConcern = false,
+                UseMajorityWriteConcern = false
+            }
+        });
+
+        return new MongoCollectionProvider(_fixture.Database, options);
+    }
+}
--- a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs
+++ b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs
@@ -13,6 +13,7 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc.Testing;
 using Microsoft.AspNetCore.TestHost;
 using Microsoft.Extensions.DependencyInjection;
+using StellaOps.Scanner.Core.Contracts;
 using StellaOps.Scanner.EntryTrace;
 using StellaOps.Scanner.EntryTrace.Serialization;
 using StellaOps.Scanner.Storage.Catalog;
@@ -365,6 +366,66 @@ public sealed class ScansEndpointsTests
        Assert.Equal(ndjson, payload.Ndjson);
    }

+    [Fact]
+    public async Task RubyPackagesEndpointReturnsNotFoundWhenMissing()
+    {
+        using var factory = new ScannerApplicationFactory();
+        using var client = factory.CreateClient();
+
+        var response = await client.GetAsync("/api/v1/scans/scan-ruby-missing/ruby-packages");
+
+        Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
+    }
+
+    [Fact]
+    public async Task RubyPackagesEndpointReturnsInventory()
+    {
+        const string scanId = "scan-ruby-existing";
+        const string digest = "sha256:feedfacefeedfacefeedfacefeedfacefeedfacefeedfacefeedfacefeedface";
+        var generatedAt = DateTime.UtcNow.AddMinutes(-10);
+
+        using var factory = new ScannerApplicationFactory();
+
+        using (var scope = factory.Services.CreateScope())
+        {
+            var repository = scope.ServiceProvider.GetRequiredService<RubyPackageInventoryRepository>();
+            var document = new RubyPackageInventoryDocument
+            {
+                ScanId = scanId,
+                ImageDigest = digest,
+                GeneratedAtUtc = generatedAt,
+                Packages = new List<RubyPackageDocument>
+                {
+                    new()
+                    {
+                        Id = "pkg:gem/rack@3.1.0",
+                        Name = "rack",
+                        Version = "3.1.0",
+                        Source = "rubygems",
+                        Platform = "ruby",
+                        Groups = new List<string> { "default" },
+                        RuntimeUsed = true,
+                        Provenance = new RubyPackageProvenance("rubygems", "Gemfile.lock", "Gemfile.lock")
+                    }
+                }
+            };
+
+            await repository.UpsertAsync(document, CancellationToken.None).ConfigureAwait(false);
+        }
+
+        using var client = factory.CreateClient();
+        var response = await client.GetAsync($"/api/v1/scans/{scanId}/ruby-packages");
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+
+        var payload = await response.Content.ReadFromJsonAsync<RubyPackagesResponse>();
+        Assert.NotNull(payload);
+        Assert.Equal(scanId, payload!.ScanId);
+        Assert.Equal(digest, payload.ImageDigest);
+        Assert.Single(payload.Packages);
+        Assert.Equal("rack", payload.Packages[0].Name);
+        Assert.Equal("rubygems", payload.Packages[0].Source);
+    }
+
    private sealed class RecordingCoordinator : IScanCoordinator
    {
        private readonly IHttpContextAccessor accessor;
--- a/src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/StellaOps.Scanner.Worker.Tests.csproj
+++ b/src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/StellaOps.Scanner.Worker.Tests.csproj
@@ -10,5 +10,6 @@
  <ItemGroup>
    <ProjectReference Include="../../StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj" />
    <ProjectReference Include="../../__Libraries/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj" />
+    <ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/StellaOps.Scanner.Analyzers.Lang.Ruby.csproj" />
  </ItemGroup>
-</Project>
+</Project>
--- a/src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/SurfaceManifestStageExecutorTests.cs
+++ b/src/Scanner/__Tests/StellaOps.Scanner.Worker.Tests/SurfaceManifestStageExecutorTests.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Collections.ObjectModel;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -11,6 +12,8 @@ using System.Threading.Tasks;
 using System.Security.Cryptography;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
+using StellaOps.Scanner.Analyzers.Lang;
+using StellaOps.Scanner.Analyzers.Lang.Ruby;
 using StellaOps.Scanner.Core.Contracts;
 using StellaOps.Scanner.EntryTrace;
 using StellaOps.Scanner.Surface.Env;
@@ -44,7 +47,8 @@ public sealed class SurfaceManifestStageExecutorTests
            environment,
            metrics,
            NullLogger<SurfaceManifestStageExecutor>.Instance,
-            hash);
+            hash,
+            new NullRubyPackageInventoryStore());

        var context = CreateContext();

@@ -80,7 +84,8 @@ public sealed class SurfaceManifestStageExecutorTests
            environment,
            metrics,
            NullLogger<SurfaceManifestStageExecutor>.Instance,
-            hash);
+            hash,
+            new NullRubyPackageInventoryStore());

        var context = CreateContext();
        PopulateAnalysis(context);
@@ -158,6 +163,69 @@ public sealed class SurfaceManifestStageExecutorTests
        context.Analysis.Set(ScanAnalysisKeys.LayerComponentFragments, ImmutableArray.Create(fragment));
    }

+    [Fact]
+    public async Task ExecuteAsync_PersistsRubyPackageInventoryWhenResultsExist()
+    {
+        var metrics = new ScannerWorkerMetrics();
+        var publisher = new TestSurfaceManifestPublisher("tenant-a");
+        var cache = new RecordingSurfaceCache();
+        var environment = new TestSurfaceEnvironment("tenant-a");
+        var hash = CreateCryptoHash();
+        var packageStore = new RecordingRubyPackageStore();
+
+        var executor = new SurfaceManifestStageExecutor(
+            publisher,
+            cache,
+            environment,
+            metrics,
+            NullLogger<SurfaceManifestStageExecutor>.Instance,
+            hash,
+            packageStore);
+
+        var context = CreateContext();
+        PopulateAnalysis(context);
+        await PopulateRubyAnalyzerResultsAsync(context);
+
+        await executor.ExecuteAsync(context, CancellationToken.None);
+
+        Assert.NotNull(packageStore.LastInventory);
+        Assert.Equal(context.ScanId, packageStore.LastInventory!.ScanId);
+        Assert.NotEmpty(packageStore.LastInventory!.Packages);
+    }
+
+    private static async Task PopulateRubyAnalyzerResultsAsync(ScanJobContext context)
+    {
+        var fixturePath = Path.Combine(
+            ResolveRepositoryRoot(),
+            "src",
+            "Scanner",
+            "__Tests",
+            "StellaOps.Scanner.Analyzers.Lang.Ruby.Tests",
+            "Fixtures",
+            "lang",
+            "ruby",
+            "simple-app");
+
+        var analyzer = new RubyLanguageAnalyzer();
+        var engine = new LanguageAnalyzerEngine(new ILanguageAnalyzer[] { analyzer });
+        var analyzerContext = new LanguageAnalyzerContext(
+            fixturePath,
+            TimeProvider.System,
+            usageHints: null,
+            services: null,
+            analysisStore: context.Analysis);
+
+        var result = await engine.AnalyzeAsync(analyzerContext, CancellationToken.None);
+        var dictionary = new Dictionary<string, LanguageAnalyzerResult>(StringComparer.OrdinalIgnoreCase)
+        {
+            ["ruby"] = result
+        };
+
+        context.Analysis.Set(
+            ScanAnalysisKeys.LanguageAnalyzerResults,
+            new ReadOnlyDictionary<string, LanguageAnalyzerResult>(dictionary));
+    }
+
    [Fact]
    public async Task ExecuteAsync_IncludesDenoObservationPayloadWhenPresent()
    {
@@ -172,7 +240,8 @@ public sealed class SurfaceManifestStageExecutorTests
            environment,
            metrics,
            NullLogger<SurfaceManifestStageExecutor>.Instance,
-            hash);
+            hash,
+            new NullRubyPackageInventoryStore());

        var context = CreateContext();
        var observationBytes = Encoding.UTF8.GetBytes("{\"entrypoints\":[\"mod.ts\"]}");
@@ -390,6 +459,36 @@ public sealed class SurfaceManifestStageExecutorTests
        }
    }

+    private sealed class RecordingRubyPackageStore : IRubyPackageInventoryStore
+    {
+        public RubyPackageInventory? LastInventory { get; private set; }
+
+        public Task StoreAsync(RubyPackageInventory inventory, CancellationToken cancellationToken)
+        {
+            LastInventory = inventory;
+            return Task.CompletedTask;
+        }
+
+        public Task<RubyPackageInventory?> GetAsync(string scanId, CancellationToken cancellationToken)
+            => Task.FromResult(LastInventory);
+    }
+
+    private static string ResolveRepositoryRoot()
+    {
+        var directory = AppContext.BaseDirectory;
+        while (!string.IsNullOrWhiteSpace(directory))
+        {
+            if (Directory.Exists(Path.Combine(directory, ".git")))
+            {
+                return directory;
+            }
+
+            directory = Path.GetDirectoryName(directory) ?? string.Empty;
+        }
+
+        throw new InvalidOperationException("Repository root not found.");
+    }
+
    private sealed class FakeJobLease : IScanJobLease
    {
        private readonly Dictionary<string, string> _metadata = new()