Add inline DSSE provenance documentation and Mongo schema
- Introduced a new document outlining the inline DSSE provenance for SBOM, VEX, scan, and derived events. - Defined the Mongo schema for event patches, including key fields for provenance and trust verification. - Documented the write path for ingesting provenance metadata and backfilling historical events. - Created CI/CD snippets for uploading DSSE attestations and generating provenance metadata. - Established Mongo indexes for efficient provenance queries and provided query recipes for various use cases. - Outlined policy gates for managing VEX decisions based on provenance verification. - Included UI nudges for displaying provenance information and implementation tasks for future enhancements. --- Implement reachability lattice and scoring model - Developed a comprehensive document detailing the reachability lattice and scoring model. - Defined core types for reachability states, evidence, and mitigations with corresponding C# models. - Established a scoring policy with base score contributions from various evidence classes. - Mapped reachability states to VEX gates and provided a clear overview of evidence sources. - Documented the event graph schema for persisting reachability data in MongoDB. - Outlined the integration of runtime probes for evidence collection and defined a roadmap for future tasks. --- Introduce uncertainty states and entropy scoring - Created a draft document for tracking uncertainty states and their impact on risk scoring. - Defined core uncertainty states with associated entropy values and evidence requirements. - Established a schema for storing uncertainty states alongside findings. - Documented the risk score calculation incorporating uncertainty and its effect on final risk assessments. - Provided policy guidelines for handling uncertainty in decision-making processes. - Outlined UI guidelines for displaying uncertainty information and suggested remediation actions. --- Add Ruby package inventory management - Implemented Ruby package inventory management with corresponding data models and storage mechanisms. - Created C# records for Ruby package inventory, artifacts, provenance, and runtime details. - Developed a repository for managing Ruby package inventory documents in MongoDB. - Implemented a service for storing and retrieving Ruby package inventories. - Added unit tests for the Ruby package inventory store to ensure functionality and data integrity.
This commit is contained in:
master
@@ -0,0 +1,54 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.Core.Contracts;
|
||||
|
||||
public sealed record RubyPackageInventory(
|
||||
string ScanId,
|
||||
string ImageDigest,
|
||||
DateTimeOffset GeneratedAtUtc,
|
||||
IReadOnlyList<RubyPackageArtifact> Packages);
|
||||
|
||||
public sealed record RubyPackageArtifact(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("name")] string Name,
|
||||
[property: JsonPropertyName("version")] string? Version,
|
||||
[property: JsonPropertyName("source")] string? Source,
|
||||
[property: JsonPropertyName("platform")] string? Platform,
|
||||
[property: JsonPropertyName("groups")] IReadOnlyList<string>? Groups,
|
||||
[property: JsonPropertyName("declaredOnly")] bool? DeclaredOnly,
|
||||
[property: JsonPropertyName("runtimeUsed")] bool? RuntimeUsed,
|
||||
[property: JsonPropertyName("provenance")] RubyPackageProvenance? Provenance,
|
||||
[property: JsonPropertyName("runtime")] RubyPackageRuntime? Runtime,
|
||||
[property: JsonPropertyName("metadata")] IReadOnlyDictionary<string, string?>? Metadata);
|
||||
|
||||
public sealed record RubyPackageProvenance(
|
||||
[property: JsonPropertyName("source")] string? Source,
|
||||
[property: JsonPropertyName("lockfile")] string? Lockfile,
|
||||
[property: JsonPropertyName("locator")] string? Locator);
|
||||
|
||||
public sealed record RubyPackageRuntime(
|
||||
[property: JsonPropertyName("entrypoints")] IReadOnlyList<string>? Entrypoints,
|
||||
[property: JsonPropertyName("files")] IReadOnlyList<string>? Files,
|
||||
[property: JsonPropertyName("reasons")] IReadOnlyList<string>? Reasons);
|
||||
|
||||
public interface IRubyPackageInventoryStore
|
||||
{
|
||||
Task StoreAsync(RubyPackageInventory inventory, CancellationToken cancellationToken);
|
||||
|
||||
Task<RubyPackageInventory?> GetAsync(string scanId, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class NullRubyPackageInventoryStore : IRubyPackageInventoryStore
|
||||
{
|
||||
public Task StoreAsync(RubyPackageInventory inventory, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inventory);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<RubyPackageInventory?> GetAsync(string scanId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
|
||||
return Task.FromResult<RubyPackageInventory?>(null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Catalog;
|
||||
|
||||
[BsonIgnoreExtraElements]
|
||||
public sealed class RubyPackageInventoryDocument
|
||||
{
|
||||
[BsonId]
|
||||
public string ScanId { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("imageDigest")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? ImageDigest { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("generatedAtUtc")]
|
||||
public DateTime GeneratedAtUtc { get; set; }
|
||||
= DateTime.UtcNow;
|
||||
|
||||
[BsonElement("packages")]
|
||||
public List<RubyPackageDocument> Packages { get; set; }
|
||||
= new();
|
||||
}
|
||||
|
||||
[BsonIgnoreExtraElements]
|
||||
public sealed class RubyPackageDocument
|
||||
{
|
||||
[BsonElement("id")]
|
||||
public string Id { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("name")]
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("version")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? Version { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("source")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? Source { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("platform")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? Platform { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("groups")]
|
||||
[BsonIgnoreIfNull]
|
||||
public List<string>? Groups { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("declaredOnly")]
|
||||
[BsonIgnoreIfNull]
|
||||
public bool? DeclaredOnly { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("runtimeUsed")]
|
||||
[BsonIgnoreIfNull]
|
||||
public bool? RuntimeUsed { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("provenance")]
|
||||
[BsonIgnoreIfNull]
|
||||
public RubyPackageProvenance? Provenance { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("runtime")]
|
||||
[BsonIgnoreIfNull]
|
||||
public RubyPackageRuntime? Runtime { get; set; }
|
||||
= null;
|
||||
|
||||
[BsonElement("metadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
= null;
|
||||
}
|
||||
@@ -9,6 +9,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.EntryTrace;
|
||||
using StellaOps.Scanner.Storage.Migrations;
|
||||
using StellaOps.Scanner.Storage.Mongo;
|
||||
@@ -67,7 +68,9 @@ public static class ServiceCollectionExtensions
|
||||
services.TryAddSingleton<LifecycleRuleRepository>();
|
||||
services.TryAddSingleton<RuntimeEventRepository>();
|
||||
services.TryAddSingleton<EntryTraceRepository>();
|
||||
services.TryAddSingleton<RubyPackageInventoryRepository>();
|
||||
services.AddSingleton<IEntryTraceResultStore, EntryTraceResultStore>();
|
||||
services.AddSingleton<IRubyPackageInventoryStore, RubyPackageInventoryStore>();
|
||||
|
||||
services.AddHttpClient(RustFsArtifactObjectStore.HttpClientName)
|
||||
.ConfigureHttpClient((sp, client) =>
|
||||
|
||||
@@ -37,15 +37,16 @@ public sealed class MongoBootstrapper
|
||||
|
||||
private async Task EnsureCollectionsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var targetCollections = new[]
|
||||
{
|
||||
ScannerStorageDefaults.Collections.Artifacts,
|
||||
ScannerStorageDefaults.Collections.Images,
|
||||
var targetCollections = new[]
|
||||
{
|
||||
ScannerStorageDefaults.Collections.Artifacts,
|
||||
ScannerStorageDefaults.Collections.Images,
|
||||
ScannerStorageDefaults.Collections.Layers,
|
||||
ScannerStorageDefaults.Collections.Links,
|
||||
ScannerStorageDefaults.Collections.Jobs,
|
||||
ScannerStorageDefaults.Collections.LifecycleRules,
|
||||
ScannerStorageDefaults.Collections.RuntimeEvents,
|
||||
ScannerStorageDefaults.Collections.RubyPackages,
|
||||
ScannerStorageDefaults.Collections.Migrations,
|
||||
};
|
||||
|
||||
@@ -66,13 +67,14 @@ public sealed class MongoBootstrapper
|
||||
|
||||
private async Task EnsureIndexesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await EnsureArtifactIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureImageIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureArtifactIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureImageIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureLayerIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureLinkIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureJobIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureLifecycleIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureRuntimeEventIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
await EnsureRubyPackageIndexesAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private Task EnsureArtifactIndexesAsync(CancellationToken cancellationToken)
|
||||
@@ -216,4 +218,20 @@ public sealed class MongoBootstrapper
|
||||
|
||||
return collection.Indexes.CreateManyAsync(models, cancellationToken);
|
||||
}
|
||||
|
||||
private Task EnsureRubyPackageIndexesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var collection = _database.GetCollection<RubyPackageInventoryDocument>(ScannerStorageDefaults.Collections.RubyPackages);
|
||||
var models = new List<CreateIndexModel<RubyPackageInventoryDocument>>
|
||||
{
|
||||
new(
|
||||
Builders<RubyPackageInventoryDocument>.IndexKeys.Ascending(x => x.ImageDigest),
|
||||
new CreateIndexOptions { Name = "rubyPackages_imageDigest", Sparse = true }),
|
||||
new(
|
||||
Builders<RubyPackageInventoryDocument>.IndexKeys.Ascending(x => x.GeneratedAtUtc),
|
||||
new CreateIndexOptions { Name = "rubyPackages_generatedAt" })
|
||||
};
|
||||
|
||||
return collection.Indexes.CreateManyAsync(models, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ public sealed class MongoCollectionProvider
|
||||
public IMongoCollection<LifecycleRuleDocument> LifecycleRules => GetCollection<LifecycleRuleDocument>(ScannerStorageDefaults.Collections.LifecycleRules);
|
||||
public IMongoCollection<RuntimeEventDocument> RuntimeEvents => GetCollection<RuntimeEventDocument>(ScannerStorageDefaults.Collections.RuntimeEvents);
|
||||
public IMongoCollection<EntryTraceDocument> EntryTrace => GetCollection<EntryTraceDocument>(ScannerStorageDefaults.Collections.EntryTrace);
|
||||
public IMongoCollection<RubyPackageInventoryDocument> RubyPackages => GetCollection<RubyPackageInventoryDocument>(ScannerStorageDefaults.Collections.RubyPackages);
|
||||
|
||||
private IMongoCollection<TDocument> GetCollection<TDocument>(string name)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Mongo;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
public sealed class RubyPackageInventoryRepository
|
||||
{
|
||||
private readonly MongoCollectionProvider _collections;
|
||||
|
||||
public RubyPackageInventoryRepository(MongoCollectionProvider collections)
|
||||
{
|
||||
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
|
||||
}
|
||||
|
||||
public async Task<RubyPackageInventoryDocument?> GetAsync(string scanId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
|
||||
return await _collections.RubyPackages
|
||||
.Find(x => x.ScanId == scanId)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task UpsertAsync(RubyPackageInventoryDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
await _collections.RubyPackages
|
||||
.ReplaceOneAsync(x => x.ScanId == document.ScanId, document, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ public static class ScannerStorageDefaults
|
||||
public const string LifecycleRules = "lifecycle_rules";
|
||||
public const string RuntimeEvents = "runtime.events";
|
||||
public const string EntryTrace = "entrytrace";
|
||||
public const string RubyPackages = "ruby.packages";
|
||||
public const string Migrations = "schema_migrations";
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Services;
|
||||
|
||||
public sealed class RubyPackageInventoryStore : IRubyPackageInventoryStore
|
||||
{
|
||||
private readonly RubyPackageInventoryRepository _repository;
|
||||
|
||||
public RubyPackageInventoryStore(RubyPackageInventoryRepository repository)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
}
|
||||
|
||||
public async Task StoreAsync(RubyPackageInventory inventory, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inventory);
|
||||
|
||||
var document = new RubyPackageInventoryDocument
|
||||
{
|
||||
ScanId = inventory.ScanId,
|
||||
ImageDigest = inventory.ImageDigest,
|
||||
GeneratedAtUtc = inventory.GeneratedAtUtc.UtcDateTime,
|
||||
Packages = inventory.Packages.Select(ToDocument).ToList()
|
||||
};
|
||||
|
||||
await _repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task<RubyPackageInventory?> GetAsync(string scanId, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
|
||||
|
||||
var document = await _repository.GetAsync(scanId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var generatedAt = DateTime.SpecifyKind(document.GeneratedAtUtc, DateTimeKind.Utc);
|
||||
var packages = document.Packages?.Select(FromDocument).ToImmutableArray()
|
||||
?? ImmutableArray<RubyPackageArtifact>.Empty;
|
||||
|
||||
return new RubyPackageInventory(
|
||||
document.ScanId,
|
||||
document.ImageDigest ?? string.Empty,
|
||||
new DateTimeOffset(generatedAt),
|
||||
packages);
|
||||
}
|
||||
|
||||
private static RubyPackageDocument ToDocument(RubyPackageArtifact artifact)
|
||||
{
|
||||
var doc = new RubyPackageDocument
|
||||
{
|
||||
Id = artifact.Id,
|
||||
Name = artifact.Name,
|
||||
Version = artifact.Version,
|
||||
Source = artifact.Source,
|
||||
Platform = artifact.Platform,
|
||||
Groups = artifact.Groups?.ToList(),
|
||||
DeclaredOnly = artifact.DeclaredOnly,
|
||||
RuntimeUsed = artifact.RuntimeUsed,
|
||||
Provenance = artifact.Provenance,
|
||||
Runtime = artifact.Runtime,
|
||||
Metadata = artifact.Metadata is null ? null : new Dictionary<string, string?>(artifact.Metadata, StringComparer.OrdinalIgnoreCase)
|
||||
};
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
private static RubyPackageArtifact FromDocument(RubyPackageDocument document)
|
||||
{
|
||||
IReadOnlyList<string>? groups = document.Groups;
|
||||
IReadOnlyDictionary<string, string?>? metadata = document.Metadata;
|
||||
|
||||
return new RubyPackageArtifact(
|
||||
document.Id,
|
||||
document.Name,
|
||||
document.Version,
|
||||
document.Source,
|
||||
document.Platform,
|
||||
groups,
|
||||
document.DeclaredOnly,
|
||||
document.RuntimeUsed,
|
||||
document.Provenance,
|
||||
document.Runtime,
|
||||
metadata);
|
||||
}
|
||||
}
|
||||
@@ -17,5 +17,6 @@
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\\StellaOps.Scanner.EntryTrace\\StellaOps.Scanner.EntryTrace.csproj" />
|
||||
<ProjectReference Include="..\\StellaOps.Scanner.Core\\StellaOps.Scanner.Core.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user