Add inline DSSE provenance documentation and Mongo schema

- Introduced a new document outlining the inline DSSE provenance for SBOM, VEX, scan, and derived events.
- Defined the Mongo schema for event patches, including key fields for provenance and trust verification.
- Documented the write path for ingesting provenance metadata and backfilling historical events.
- Created CI/CD snippets for uploading DSSE attestations and generating provenance metadata.
- Established Mongo indexes for efficient provenance queries and provided query recipes for various use cases.
- Outlined policy gates for managing VEX decisions based on provenance verification.
- Included UI nudges for displaying provenance information and implementation tasks for future enhancements.

---

Implement reachability lattice and scoring model

- Developed a comprehensive document detailing the reachability lattice and scoring model.
- Defined core types for reachability states, evidence, and mitigations with corresponding C# models.
- Established a scoring policy with base score contributions from various evidence classes.
- Mapped reachability states to VEX gates and provided a clear overview of evidence sources.
- Documented the event graph schema for persisting reachability data in MongoDB.
- Outlined the integration of runtime probes for evidence collection and defined a roadmap for future tasks.

---

Introduce uncertainty states and entropy scoring

- Created a draft document for tracking uncertainty states and their impact on risk scoring.
- Defined core uncertainty states with associated entropy values and evidence requirements.
- Established a schema for storing uncertainty states alongside findings.
- Documented the risk score calculation incorporating uncertainty and its effect on final risk assessments.
- Provided policy guidelines for handling uncertainty in decision-making processes.
- Outlined UI guidelines for displaying uncertainty information and suggested remediation actions.

---

Add Ruby package inventory management

- Implemented Ruby package inventory management with corresponding data models and storage mechanisms.
- Created C# records for Ruby package inventory, artifacts, provenance, and runtime details.
- Developed a repository for managing Ruby package inventory documents in MongoDB.
- Implemented a service for storing and retrieving Ruby package inventories.
- Added unit tests for the Ruby package inventory store to ensure functionality and data integrity.
This commit is contained in:
master
2025-11-13 00:20:33 +02:00
parent 86be324fc0
commit 7040984215
41 changed files with 1955 additions and 76 deletions

View File

@@ -0,0 +1,21 @@
using System.Text.Json.Serialization;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.WebService.Contracts;
public sealed record RubyPackagesResponse
{
[JsonPropertyName("scanId")]
public string ScanId { get; init; } = string.Empty;
[JsonPropertyName("imageDigest")]
public string ImageDigest { get; init; } = string.Empty;
[JsonPropertyName("generatedAt")]
public DateTimeOffset GeneratedAt { get; init; }
= DateTimeOffset.UtcNow;
[JsonPropertyName("packages")]
public IReadOnlyList<RubyPackageArtifact> Packages { get; init; }
= Array.Empty<RubyPackageArtifact>();
}

View File

@@ -13,6 +13,8 @@ using StellaOps.Scanner.WebService.Domain;
using StellaOps.Scanner.WebService.Infrastructure;
using StellaOps.Scanner.WebService.Security;
using StellaOps.Scanner.WebService.Services;
using DomainScanProgressEvent = StellaOps.Scanner.WebService.Domain.ScanProgressEvent;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
namespace StellaOps.Scanner.WebService.Endpoints;
@@ -54,6 +56,12 @@ internal static class ScanEndpoints
.Produces<EntryTraceResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(ScannerPolicies.ScansRead);
scans.MapGet("/{scanId}/ruby-packages", HandleRubyPackagesAsync)
.WithName("scanner.scans.ruby-packages")
.Produces<RubyPackagesResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.RequireAuthorization(ScannerPolicies.ScansRead);
}
private static async Task<IResult> HandleSubmitAsync(
@@ -311,6 +319,46 @@ internal static class ScanEndpoints
return Json(response, StatusCodes.Status200OK);
}
private static async Task<IResult> HandleRubyPackagesAsync(
string scanId,
IRubyPackageInventoryStore inventoryStore,
HttpContext context,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(inventoryStore);
if (!ScanId.TryParse(scanId, out var parsed))
{
return ProblemResultFactory.Create(
context,
ProblemTypes.Validation,
"Invalid scan identifier",
StatusCodes.Status400BadRequest,
detail: "Scan identifier is required.");
}
var inventory = await inventoryStore.GetAsync(parsed.Value, cancellationToken).ConfigureAwait(false);
if (inventory is null)
{
return ProblemResultFactory.Create(
context,
ProblemTypes.NotFound,
"Ruby packages not found",
StatusCodes.Status404NotFound,
detail: "Ruby package inventory is not available for the requested scan.");
}
var response = new RubyPackagesResponse
{
ScanId = inventory.ScanId,
ImageDigest = inventory.ImageDigest,
GeneratedAt = inventory.GeneratedAtUtc,
Packages = inventory.Packages
};
return Json(response, StatusCodes.Status200OK);
}
private static IReadOnlyDictionary<string, string> NormalizeMetadata(IDictionary<string, string> metadata)
{
if (metadata is null || metadata.Count == 0)
@@ -342,7 +390,7 @@ internal static class ScanEndpoints
await writer.WriteAsync(new[] { (byte)'\n' }, cancellationToken).ConfigureAwait(false);
}
private static async Task WriteSseAsync(PipeWriter writer, object payload, ScanProgressEvent progressEvent, CancellationToken cancellationToken)
private static async Task WriteSseAsync(PipeWriter writer, object payload, DomainScanProgressEvent progressEvent, CancellationToken cancellationToken)
{
var json = JsonSerializer.Serialize(payload, SerializerOptions);
var eventName = progressEvent.State.ToLowerInvariant();

View File

@@ -19,6 +19,7 @@ using StellaOps.Cryptography.DependencyInjection;
using StellaOps.Cryptography.Plugin.BouncyCastle;
using StellaOps.Policy;
using StellaOps.Scanner.Cache;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.FS;
using StellaOps.Scanner.Surface.Secrets;

View File

@@ -13,6 +13,7 @@ using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.FS;
using StellaOps.Scanner.WebService.Contracts;
using StellaOps.Scanner.WebService.Options;

View File

@@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StellaOps.Scanner.Analyzers.Lang;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Worker.Processing.Surface;
internal static class RubyPackageInventoryBuilder
{
private const string AnalyzerId = "ruby";
public static IReadOnlyList<RubyPackageArtifact> Build(LanguageAnalyzerResult result)
{
ArgumentNullException.ThrowIfNull(result);
var artifacts = new List<RubyPackageArtifact>();
foreach (var component in result.Components)
{
if (!component.AnalyzerId.Equals(AnalyzerId, StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (!string.Equals(component.Type, "gem", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var metadata = component.Metadata ?? new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
var metadataCopy = new Dictionary<string, string?>(metadata, StringComparer.OrdinalIgnoreCase);
var groups = SplitList(metadataCopy, "groups");
var entrypoints = SplitList(metadataCopy, "runtime.entrypoints");
var runtimeFiles = SplitList(metadataCopy, "runtime.files");
var runtimeReasons = SplitList(metadataCopy, "runtime.reasons");
var declaredOnly = TryParseBool(metadataCopy, "declaredOnly");
var runtimeUsed = TryParseBool(metadataCopy, "runtime.used") ?? component.UsedByEntrypoint;
var source = GetString(metadataCopy, "source");
var platform = GetString(metadataCopy, "platform");
var lockfile = GetString(metadataCopy, "lockfile");
var artifactLocator = GetString(metadataCopy, "artifact");
var provenance = (source is not null || lockfile is not null || artifactLocator is not null)
? new RubyPackageProvenance(source, lockfile, artifactLocator ?? lockfile)
: null;
RubyPackageRuntime? runtime = null;
if (entrypoints is { Count: > 0 } || runtimeFiles is { Count: > 0 } || runtimeReasons is { Count: > 0 })
{
runtime = new RubyPackageRuntime(entrypoints, runtimeFiles, runtimeReasons);
}
artifacts.Add(new RubyPackageArtifact(
component.ComponentKey,
component.Name,
component.Version,
source,
platform,
groups,
declaredOnly,
runtimeUsed,
provenance,
runtime,
metadataCopy));
}
return artifacts;
}
private static IReadOnlyList<string>? SplitList(IReadOnlyDictionary<string, string?> metadata, string key)
{
if (!metadata.TryGetValue(key, out var raw) || string.IsNullOrWhiteSpace(raw))
{
return Array.Empty<string>();
}
var values = raw
.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Where(static value => !string.IsNullOrWhiteSpace(value))
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
return values.Length == 0 ? Array.Empty<string>() : values;
}
private static bool? TryParseBool(IReadOnlyDictionary<string, string?> metadata, string key)
{
if (!metadata.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value))
{
return null;
}
if (bool.TryParse(value, out var parsed))
{
return parsed;
}
return null;
}
private static string? GetString(IReadOnlyDictionary<string, string?> metadata, string key)
{
if (!metadata.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value))
{
return null;
}
return value.Trim();
}
}

View File

@@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Globalization;
using System.Reflection;
@@ -7,6 +8,7 @@ using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Analyzers.Lang;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.EntryTrace.Serialization;
@@ -38,6 +40,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
private readonly ScannerWorkerMetrics _metrics;
private readonly ILogger<SurfaceManifestStageExecutor> _logger;
private readonly ICryptoHash _hash;
private readonly IRubyPackageInventoryStore _rubyPackageStore;
private readonly string _componentVersion;
public SurfaceManifestStageExecutor(
@@ -46,7 +49,8 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
ISurfaceEnvironment surfaceEnvironment,
ScannerWorkerMetrics metrics,
ILogger<SurfaceManifestStageExecutor> logger,
ICryptoHash hash)
ICryptoHash hash,
IRubyPackageInventoryStore rubyPackageStore)
{
_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
_surfaceCache = surfaceCache ?? throw new ArgumentNullException(nameof(surfaceCache));
@@ -54,6 +58,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_hash = hash ?? throw new ArgumentNullException(nameof(hash));
_rubyPackageStore = rubyPackageStore ?? throw new ArgumentNullException(nameof(rubyPackageStore));
_componentVersion = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown";
}
@@ -64,6 +69,7 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
ArgumentNullException.ThrowIfNull(context);
var payloads = CollectPayloads(context);
await PersistRubyPackagesAsync(context, cancellationToken).ConfigureAwait(false);
if (payloads.Count == 0)
{
_metrics.RecordSurfaceManifestSkipped(context);
@@ -182,6 +188,33 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
return payloads;
}
private async Task PersistRubyPackagesAsync(ScanJobContext context, CancellationToken cancellationToken)
{
if (!context.Analysis.TryGet<ReadOnlyDictionary<string, LanguageAnalyzerResult>>(ScanAnalysisKeys.LanguageAnalyzerResults, out var results))
{
return;
}
if (!results.TryGetValue("ruby", out var rubyResult) || rubyResult is null)
{
return;
}
var packages = RubyPackageInventoryBuilder.Build(rubyResult);
if (packages.Count == 0)
{
return;
}
var inventory = new RubyPackageInventory(
context.ScanId,
ResolveImageDigest(context),
context.TimeProvider.GetUtcNow(),
packages);
await _rubyPackageStore.StoreAsync(inventory, cancellationToken).ConfigureAwait(false);
}
private async Task PersistPayloadsToSurfaceCacheAsync(
ScanJobContext context,
string tenant,

View File

@@ -1,15 +1,16 @@
using System.Diagnostics;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Auth.Client;
using StellaOps.Configuration;
using StellaOps.Scanner.Cache;
using StellaOps.Scanner.Analyzers.OS.Plugin;
using StellaOps.Scanner.Analyzers.Lang.Plugin;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Security;
using StellaOps.Scanner.Surface.Env;
using StellaOps.Scanner.Surface.FS;
@@ -59,6 +60,10 @@ if (!string.IsNullOrWhiteSpace(connectionString))
builder.Services.AddSingleton<ISurfaceManifestPublisher, SurfaceManifestPublisher>();
builder.Services.AddSingleton<IScanStageExecutor, SurfaceManifestStageExecutor>();
}
else
{
builder.Services.TryAddSingleton<IRubyPackageInventoryStore, NullRubyPackageInventoryStore>();
}
builder.Services.TryAddSingleton<IScanJobSource, NullScanJobSource>();
builder.Services.TryAddSingleton<IPluginCatalogGuard, RestartOnlyPluginGuard>();

View File

@@ -0,0 +1,54 @@
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Core.Contracts;
public sealed record RubyPackageInventory(
string ScanId,
string ImageDigest,
DateTimeOffset GeneratedAtUtc,
IReadOnlyList<RubyPackageArtifact> Packages);
public sealed record RubyPackageArtifact(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("name")] string Name,
[property: JsonPropertyName("version")] string? Version,
[property: JsonPropertyName("source")] string? Source,
[property: JsonPropertyName("platform")] string? Platform,
[property: JsonPropertyName("groups")] IReadOnlyList<string>? Groups,
[property: JsonPropertyName("declaredOnly")] bool? DeclaredOnly,
[property: JsonPropertyName("runtimeUsed")] bool? RuntimeUsed,
[property: JsonPropertyName("provenance")] RubyPackageProvenance? Provenance,
[property: JsonPropertyName("runtime")] RubyPackageRuntime? Runtime,
[property: JsonPropertyName("metadata")] IReadOnlyDictionary<string, string?>? Metadata);
public sealed record RubyPackageProvenance(
[property: JsonPropertyName("source")] string? Source,
[property: JsonPropertyName("lockfile")] string? Lockfile,
[property: JsonPropertyName("locator")] string? Locator);
public sealed record RubyPackageRuntime(
[property: JsonPropertyName("entrypoints")] IReadOnlyList<string>? Entrypoints,
[property: JsonPropertyName("files")] IReadOnlyList<string>? Files,
[property: JsonPropertyName("reasons")] IReadOnlyList<string>? Reasons);
public interface IRubyPackageInventoryStore
{
Task StoreAsync(RubyPackageInventory inventory, CancellationToken cancellationToken);
Task<RubyPackageInventory?> GetAsync(string scanId, CancellationToken cancellationToken);
}
public sealed class NullRubyPackageInventoryStore : IRubyPackageInventoryStore
{
public Task StoreAsync(RubyPackageInventory inventory, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(inventory);
return Task.CompletedTask;
}
public Task<RubyPackageInventory?> GetAsync(string scanId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
return Task.FromResult<RubyPackageInventory?>(null);
}
}

View File

@@ -0,0 +1,79 @@
using MongoDB.Bson.Serialization.Attributes;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Storage.Catalog;
[BsonIgnoreExtraElements]
public sealed class RubyPackageInventoryDocument
{
[BsonId]
public string ScanId { get; set; } = string.Empty;
[BsonElement("imageDigest")]
[BsonIgnoreIfNull]
public string? ImageDigest { get; set; }
= null;
[BsonElement("generatedAtUtc")]
public DateTime GeneratedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("packages")]
public List<RubyPackageDocument> Packages { get; set; }
= new();
}
[BsonIgnoreExtraElements]
public sealed class RubyPackageDocument
{
[BsonElement("id")]
public string Id { get; set; } = string.Empty;
[BsonElement("name")]
public string Name { get; set; } = string.Empty;
[BsonElement("version")]
[BsonIgnoreIfNull]
public string? Version { get; set; }
= null;
[BsonElement("source")]
[BsonIgnoreIfNull]
public string? Source { get; set; }
= null;
[BsonElement("platform")]
[BsonIgnoreIfNull]
public string? Platform { get; set; }
= null;
[BsonElement("groups")]
[BsonIgnoreIfNull]
public List<string>? Groups { get; set; }
= null;
[BsonElement("declaredOnly")]
[BsonIgnoreIfNull]
public bool? DeclaredOnly { get; set; }
= null;
[BsonElement("runtimeUsed")]
[BsonIgnoreIfNull]
public bool? RuntimeUsed { get; set; }
= null;
[BsonElement("provenance")]
[BsonIgnoreIfNull]
public RubyPackageProvenance? Provenance { get; set; }
= null;
[BsonElement("runtime")]
[BsonIgnoreIfNull]
public RubyPackageRuntime? Runtime { get; set; }
= null;
[BsonElement("metadata")]
[BsonIgnoreIfNull]
public Dictionary<string, string?>? Metadata { get; set; }
= null;
}

View File

@@ -9,6 +9,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.Storage.Migrations;
using StellaOps.Scanner.Storage.Mongo;
@@ -67,7 +68,9 @@ public static class ServiceCollectionExtensions
services.TryAddSingleton<LifecycleRuleRepository>();
services.TryAddSingleton<RuntimeEventRepository>();
services.TryAddSingleton<EntryTraceRepository>();
services.TryAddSingleton<RubyPackageInventoryRepository>();
services.AddSingleton<IEntryTraceResultStore, EntryTraceResultStore>();
services.AddSingleton<IRubyPackageInventoryStore, RubyPackageInventoryStore>();
services.AddHttpClient(RustFsArtifactObjectStore.HttpClientName)
.ConfigureHttpClient((sp, client) =>

View File

@@ -37,15 +37,16 @@ public sealed class MongoBootstrapper
private async Task EnsureCollectionsAsync(CancellationToken cancellationToken)
{
var targetCollections = new[]
{
ScannerStorageDefaults.Collections.Artifacts,
ScannerStorageDefaults.Collections.Images,
var targetCollections = new[]
{
ScannerStorageDefaults.Collections.Artifacts,
ScannerStorageDefaults.Collections.Images,
ScannerStorageDefaults.Collections.Layers,
ScannerStorageDefaults.Collections.Links,
ScannerStorageDefaults.Collections.Jobs,
ScannerStorageDefaults.Collections.LifecycleRules,
ScannerStorageDefaults.Collections.RuntimeEvents,
ScannerStorageDefaults.Collections.RubyPackages,
ScannerStorageDefaults.Collections.Migrations,
};
@@ -66,13 +67,14 @@ public sealed class MongoBootstrapper
private async Task EnsureIndexesAsync(CancellationToken cancellationToken)
{
await EnsureArtifactIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureImageIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureArtifactIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureImageIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureLayerIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureLinkIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureJobIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureLifecycleIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureRuntimeEventIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureRubyPackageIndexesAsync(cancellationToken).ConfigureAwait(false);
}
private Task EnsureArtifactIndexesAsync(CancellationToken cancellationToken)
@@ -216,4 +218,20 @@ public sealed class MongoBootstrapper
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
private Task EnsureRubyPackageIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<RubyPackageInventoryDocument>(ScannerStorageDefaults.Collections.RubyPackages);
var models = new List<CreateIndexModel<RubyPackageInventoryDocument>>
{
new(
Builders<RubyPackageInventoryDocument>.IndexKeys.Ascending(x => x.ImageDigest),
new CreateIndexOptions { Name = "rubyPackages_imageDigest", Sparse = true }),
new(
Builders<RubyPackageInventoryDocument>.IndexKeys.Ascending(x => x.GeneratedAtUtc),
new CreateIndexOptions { Name = "rubyPackages_generatedAt" })
};
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
}

View File

@@ -23,6 +23,7 @@ public sealed class MongoCollectionProvider
public IMongoCollection<LifecycleRuleDocument> LifecycleRules => GetCollection<LifecycleRuleDocument>(ScannerStorageDefaults.Collections.LifecycleRules);
public IMongoCollection<RuntimeEventDocument> RuntimeEvents => GetCollection<RuntimeEventDocument>(ScannerStorageDefaults.Collections.RuntimeEvents);
public IMongoCollection<EntryTraceDocument> EntryTrace => GetCollection<EntryTraceDocument>(ScannerStorageDefaults.Collections.EntryTrace);
public IMongoCollection<RubyPackageInventoryDocument> RubyPackages => GetCollection<RubyPackageInventoryDocument>(ScannerStorageDefaults.Collections.RubyPackages);
private IMongoCollection<TDocument> GetCollection<TDocument>(string name)
{

View File

@@ -0,0 +1,33 @@
using MongoDB.Driver;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class RubyPackageInventoryRepository
{
private readonly MongoCollectionProvider _collections;
public RubyPackageInventoryRepository(MongoCollectionProvider collections)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
}
public async Task<RubyPackageInventoryDocument?> GetAsync(string scanId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
return await _collections.RubyPackages
.Find(x => x.ScanId == scanId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
}
public async Task UpsertAsync(RubyPackageInventoryDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var options = new ReplaceOptions { IsUpsert = true };
await _collections.RubyPackages
.ReplaceOneAsync(x => x.ScanId == document.ScanId, document, options, cancellationToken)
.ConfigureAwait(false);
}
}

View File

@@ -23,6 +23,7 @@ public static class ScannerStorageDefaults
public const string LifecycleRules = "lifecycle_rules";
public const string RuntimeEvents = "runtime.events";
public const string EntryTrace = "entrytrace";
public const string RubyPackages = "ruby.packages";
public const string Migrations = "schema_migrations";
}

View File

@@ -0,0 +1,92 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Repositories;
namespace StellaOps.Scanner.Storage.Services;
public sealed class RubyPackageInventoryStore : IRubyPackageInventoryStore
{
private readonly RubyPackageInventoryRepository _repository;
public RubyPackageInventoryStore(RubyPackageInventoryRepository repository)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
}
public async Task StoreAsync(RubyPackageInventory inventory, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(inventory);
var document = new RubyPackageInventoryDocument
{
ScanId = inventory.ScanId,
ImageDigest = inventory.ImageDigest,
GeneratedAtUtc = inventory.GeneratedAtUtc.UtcDateTime,
Packages = inventory.Packages.Select(ToDocument).ToList()
};
await _repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
}
public async Task<RubyPackageInventory?> GetAsync(string scanId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
var document = await _repository.GetAsync(scanId, cancellationToken).ConfigureAwait(false);
if (document is null)
{
return null;
}
var generatedAt = DateTime.SpecifyKind(document.GeneratedAtUtc, DateTimeKind.Utc);
var packages = document.Packages?.Select(FromDocument).ToImmutableArray()
?? ImmutableArray<RubyPackageArtifact>.Empty;
return new RubyPackageInventory(
document.ScanId,
document.ImageDigest ?? string.Empty,
new DateTimeOffset(generatedAt),
packages);
}
private static RubyPackageDocument ToDocument(RubyPackageArtifact artifact)
{
var doc = new RubyPackageDocument
{
Id = artifact.Id,
Name = artifact.Name,
Version = artifact.Version,
Source = artifact.Source,
Platform = artifact.Platform,
Groups = artifact.Groups?.ToList(),
DeclaredOnly = artifact.DeclaredOnly,
RuntimeUsed = artifact.RuntimeUsed,
Provenance = artifact.Provenance,
Runtime = artifact.Runtime,
Metadata = artifact.Metadata is null ? null : new Dictionary<string, string?>(artifact.Metadata, StringComparer.OrdinalIgnoreCase)
};
return doc;
}
private static RubyPackageArtifact FromDocument(RubyPackageDocument document)
{
IReadOnlyList<string>? groups = document.Groups;
IReadOnlyDictionary<string, string?>? metadata = document.Metadata;
return new RubyPackageArtifact(
document.Id,
document.Name,
document.Version,
document.Source,
document.Platform,
groups,
document.DeclaredOnly,
document.RuntimeUsed,
document.Provenance,
document.Runtime,
metadata);
}
}

View File

@@ -17,5 +17,6 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\\StellaOps.Scanner.EntryTrace\\StellaOps.Scanner.EntryTrace.csproj" />
<ProjectReference Include="..\\StellaOps.Scanner.Core\\StellaOps.Scanner.Core.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,121 @@
using System.Collections.Generic;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Storage;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.Storage.Services;
using StellaOps.Scanner.Storage.Catalog;
using Xunit;
namespace StellaOps.Scanner.Storage.Tests;
public sealed class RubyPackageInventoryStoreTests : IClassFixture<ScannerMongoFixture>
{
private readonly ScannerMongoFixture _fixture;
public RubyPackageInventoryStoreTests(ScannerMongoFixture fixture)
{
_fixture = fixture;
}
[Fact]
public async Task StoreAsync_ThrowsWhenInventoryNull()
{
var store = CreateStore();
await Assert.ThrowsAsync<ArgumentNullException>(async () =>
{
RubyPackageInventory? inventory = null;
await store.StoreAsync(inventory!, CancellationToken.None);
});
}
[Fact]
public async Task GetAsync_ReturnsNullWhenMissing()
{
await ClearCollectionAsync();
var store = CreateStore();
var inventory = await store.GetAsync("scan-missing", CancellationToken.None);
Assert.Null(inventory);
}
[Fact]
public async Task StoreAsync_RoundTripsInventory()
{
await ClearCollectionAsync();
var store = CreateStore();
var scanId = $"scan-{Guid.NewGuid():n}";
var generatedAt = new DateTimeOffset(2025, 11, 12, 16, 10, 0, TimeSpan.Zero);
var packages = new[]
{
new RubyPackageArtifact(
Id: "purl::pkg:gem/rack@3.1.2",
Name: "rack",
Version: "3.1.2",
Source: "rubygems",
Platform: "ruby",
Groups: new[] {"default"},
DeclaredOnly: true,
RuntimeUsed: true,
Provenance: new RubyPackageProvenance("rubygems", "Gemfile.lock", "Gemfile.lock"),
Runtime: new RubyPackageRuntime(
new[] { "config.ru" },
new[] { "config.ru" },
new[] { "require-static" }),
Metadata: new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase)
{
["source"] = "rubygems",
["lockfile"] = "Gemfile.lock",
["groups"] = "default"
})
};
var inventory = new RubyPackageInventory(scanId, "sha256:image", generatedAt, packages);
await store.StoreAsync(inventory, CancellationToken.None);
var stored = await store.GetAsync(scanId, CancellationToken.None);
Assert.NotNull(stored);
Assert.Equal(scanId, stored!.ScanId);
Assert.Equal("sha256:image", stored.ImageDigest);
Assert.Equal(generatedAt, stored.GeneratedAtUtc);
Assert.Single(stored.Packages);
Assert.Equal("rack", stored.Packages[0].Name);
Assert.Equal("rubygems", stored.Packages[0].Source);
}
private async Task ClearCollectionAsync()
{
var provider = CreateProvider();
await provider.RubyPackages.DeleteManyAsync(Builders<RubyPackageInventoryDocument>.Filter.Empty);
}
private RubyPackageInventoryStore CreateStore()
{
var provider = CreateProvider();
var repository = new RubyPackageInventoryRepository(provider);
return new RubyPackageInventoryStore(repository);
}
private MongoCollectionProvider CreateProvider()
{
var options = Options.Create(new ScannerStorageOptions
{
Mongo = new MongoOptions
{
ConnectionString = _fixture.Runner.ConnectionString,
DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName,
UseMajorityReadConcern = false,
UseMajorityWriteConcern = false
}
});
return new MongoCollectionProvider(_fixture.Database, options);
}
}

View File

@@ -13,6 +13,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.EntryTrace.Serialization;
using StellaOps.Scanner.Storage.Catalog;
@@ -365,6 +366,66 @@ public sealed class ScansEndpointsTests
Assert.Equal(ndjson, payload.Ndjson);
}
[Fact]
public async Task RubyPackagesEndpointReturnsNotFoundWhenMissing()
{
using var factory = new ScannerApplicationFactory();
using var client = factory.CreateClient();
var response = await client.GetAsync("/api/v1/scans/scan-ruby-missing/ruby-packages");
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
}
[Fact]
public async Task RubyPackagesEndpointReturnsInventory()
{
const string scanId = "scan-ruby-existing";
const string digest = "sha256:feedfacefeedfacefeedfacefeedfacefeedfacefeedfacefeedfacefeedface";
var generatedAt = DateTime.UtcNow.AddMinutes(-10);
using var factory = new ScannerApplicationFactory();
using (var scope = factory.Services.CreateScope())
{
var repository = scope.ServiceProvider.GetRequiredService<RubyPackageInventoryRepository>();
var document = new RubyPackageInventoryDocument
{
ScanId = scanId,
ImageDigest = digest,
GeneratedAtUtc = generatedAt,
Packages = new List<RubyPackageDocument>
{
new()
{
Id = "pkg:gem/rack@3.1.0",
Name = "rack",
Version = "3.1.0",
Source = "rubygems",
Platform = "ruby",
Groups = new List<string> { "default" },
RuntimeUsed = true,
Provenance = new RubyPackageProvenance("rubygems", "Gemfile.lock", "Gemfile.lock")
}
}
};
await repository.UpsertAsync(document, CancellationToken.None).ConfigureAwait(false);
}
using var client = factory.CreateClient();
var response = await client.GetAsync($"/api/v1/scans/{scanId}/ruby-packages");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var payload = await response.Content.ReadFromJsonAsync<RubyPackagesResponse>();
Assert.NotNull(payload);
Assert.Equal(scanId, payload!.ScanId);
Assert.Equal(digest, payload.ImageDigest);
Assert.Single(payload.Packages);
Assert.Equal("rack", payload.Packages[0].Name);
Assert.Equal("rubygems", payload.Packages[0].Source);
}
private sealed class RecordingCoordinator : IScanCoordinator
{
private readonly IHttpContextAccessor accessor;

View File

@@ -10,5 +10,6 @@
<ItemGroup>
<ProjectReference Include="../../StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/StellaOps.Scanner.Analyzers.Lang.Ruby.csproj" />
</ItemGroup>
</Project>
</Project>

View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Text;
@@ -11,6 +12,8 @@ using System.Threading.Tasks;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Analyzers.Lang;
using StellaOps.Scanner.Analyzers.Lang.Ruby;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.Surface.Env;
@@ -44,7 +47,8 @@ public sealed class SurfaceManifestStageExecutorTests
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash);
hash,
new NullRubyPackageInventoryStore());
var context = CreateContext();
@@ -80,7 +84,8 @@ public sealed class SurfaceManifestStageExecutorTests
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash);
hash,
new NullRubyPackageInventoryStore());
var context = CreateContext();
PopulateAnalysis(context);
@@ -158,6 +163,69 @@ public sealed class SurfaceManifestStageExecutorTests
context.Analysis.Set(ScanAnalysisKeys.LayerComponentFragments, ImmutableArray.Create(fragment));
}
[Fact]
public async Task ExecuteAsync_PersistsRubyPackageInventoryWhenResultsExist()
{
var metrics = new ScannerWorkerMetrics();
var publisher = new TestSurfaceManifestPublisher("tenant-a");
var cache = new RecordingSurfaceCache();
var environment = new TestSurfaceEnvironment("tenant-a");
var hash = CreateCryptoHash();
var packageStore = new RecordingRubyPackageStore();
var executor = new SurfaceManifestStageExecutor(
publisher,
cache,
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash,
packageStore);
var context = CreateContext();
PopulateAnalysis(context);
await PopulateRubyAnalyzerResultsAsync(context);
await executor.ExecuteAsync(context, CancellationToken.None);
Assert.NotNull(packageStore.LastInventory);
Assert.Equal(context.ScanId, packageStore.LastInventory!.ScanId);
Assert.NotEmpty(packageStore.LastInventory!.Packages);
}
private static async Task PopulateRubyAnalyzerResultsAsync(ScanJobContext context)
{
var fixturePath = Path.Combine(
ResolveRepositoryRoot(),
"src",
"Scanner",
"__Tests",
"StellaOps.Scanner.Analyzers.Lang.Ruby.Tests",
"Fixtures",
"lang",
"ruby",
"simple-app");
var analyzer = new RubyLanguageAnalyzer();
var engine = new LanguageAnalyzerEngine(new ILanguageAnalyzer[] { analyzer });
var analyzerContext = new LanguageAnalyzerContext(
fixturePath,
TimeProvider.System,
usageHints: null,
services: null,
analysisStore: context.Analysis);
var result = await engine.AnalyzeAsync(analyzerContext, CancellationToken.None);
var dictionary = new Dictionary<string, LanguageAnalyzerResult>(StringComparer.OrdinalIgnoreCase)
{
["ruby"] = result
};
context.Analysis.Set(
ScanAnalysisKeys.LanguageAnalyzerResults,
new ReadOnlyDictionary<string, LanguageAnalyzerResult>(dictionary));
}
[Fact]
public async Task ExecuteAsync_IncludesDenoObservationPayloadWhenPresent()
{
@@ -172,7 +240,8 @@ public sealed class SurfaceManifestStageExecutorTests
environment,
metrics,
NullLogger<SurfaceManifestStageExecutor>.Instance,
hash);
hash,
new NullRubyPackageInventoryStore());
var context = CreateContext();
var observationBytes = Encoding.UTF8.GetBytes("{\"entrypoints\":[\"mod.ts\"]}");
@@ -390,6 +459,36 @@ public sealed class SurfaceManifestStageExecutorTests
}
}
private sealed class RecordingRubyPackageStore : IRubyPackageInventoryStore
{
public RubyPackageInventory? LastInventory { get; private set; }
public Task StoreAsync(RubyPackageInventory inventory, CancellationToken cancellationToken)
{
LastInventory = inventory;
return Task.CompletedTask;
}
public Task<RubyPackageInventory?> GetAsync(string scanId, CancellationToken cancellationToken)
=> Task.FromResult(LastInventory);
}
private static string ResolveRepositoryRoot()
{
var directory = AppContext.BaseDirectory;
while (!string.IsNullOrWhiteSpace(directory))
{
if (Directory.Exists(Path.Combine(directory, ".git")))
{
return directory;
}
directory = Path.GetDirectoryName(directory) ?? string.Empty;
}
throw new InvalidOperationException("Repository root not found.");
}
private sealed class FakeJobLease : IScanJobLease
{
private readonly Dictionary<string, string> _metadata = new()