up
Some checks failed
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Manifest Integrity / Validate Schema Integrity (push) Has been cancelled
Manifest Integrity / Validate Contract Documents (push) Has been cancelled
Manifest Integrity / Validate Pack Fixtures (push) Has been cancelled
Manifest Integrity / Audit SHA256SUMS Files (push) Has been cancelled
Manifest Integrity / Verify Merkle Roots (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-12 09:35:37 +02:00
parent ce5ec9c158
commit efaf3cb789
238 changed files with 146274 additions and 5767 deletions

View File

@@ -46,7 +46,9 @@ public sealed class ReachabilityGraphBuilder
int? sourceLine = null,
IReadOnlyDictionary<string, string>? attributes = null,
string? purl = null,
string? symbolDigest = null)
string? symbolDigest = null,
ReachabilitySymbol? symbol = null,
string? codeBlockHash = null)
{
if (string.IsNullOrWhiteSpace(symbolId))
{
@@ -63,7 +65,9 @@ public sealed class ReachabilityGraphBuilder
sourceLine,
attributes?.ToImmutableSortedDictionary(StringComparer.Ordinal) ?? ImmutableSortedDictionary<string, string>.Empty,
purl?.Trim(),
symbolDigest?.Trim());
symbolDigest?.Trim(),
symbol?.Trimmed(),
codeBlockHash?.Trim());
_richNodes[id] = node;
nodes.Add(id);
@@ -184,6 +188,8 @@ public sealed class ReachabilityGraphBuilder
rich.Lang,
rich.Kind,
rich.Display,
rich.CodeBlockHash,
rich.Symbol,
source,
rich.Attributes.Count > 0 ? rich.Attributes : null,
rich.Purl,
@@ -337,7 +343,9 @@ public sealed class ReachabilityGraphBuilder
int? SourceLine,
ImmutableSortedDictionary<string, string> Attributes,
string? Purl = null,
string? SymbolDigest = null);
string? SymbolDigest = null,
ReachabilitySymbol? Symbol = null,
string? CodeBlockHash = null);
private sealed record RichEdge(
string From,

View File

@@ -48,6 +48,7 @@ public sealed class ReachabilityReplayWriter
Kind = graph.Kind,
CasUri = graph.CasUri,
Sha256 = graph.Sha256,
HashAlgorithm = string.IsNullOrWhiteSpace(graph.Sha256) ? "sha256" : "blake3-256",
Analyzer = graph.Analyzer,
Version = graph.Version
});
@@ -81,6 +82,7 @@ public sealed class ReachabilityReplayWriter
Source = trace.Source,
CasUri = trace.CasUri,
Sha256 = trace.Sha256,
HashAlgorithm = string.IsNullOrWhiteSpace(trace.Sha256) ? "sha256" : "sha256",
RecordedAt = trace.RecordedAt
});
}

View File

@@ -0,0 +1,37 @@
using System;
namespace StellaOps.Scanner.Reachability;
/// <summary>
/// Represents optional symbol metadata (mangled/demangled names, source, confidence).
/// Used to enrich reachability evidence without altering canonical IDs.
/// </summary>
public sealed record ReachabilitySymbol(
string? Mangled,
string? Demangled,
string? Source,
double? Confidence)
{
public ReachabilitySymbol Trimmed()
=> new(
TrimOrNull(Mangled),
TrimOrNull(Demangled),
NormalizeSource(Source),
Confidence is null ? null : ClampConfidence(Confidence.Value));
private static string? TrimOrNull(string? value)
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
private static string? NormalizeSource(string? source)
{
if (string.IsNullOrWhiteSpace(source))
{
return null;
}
return source.Trim().ToUpperInvariant();
}
private static double ClampConfidence(double value)
=> Math.Max(0.0, Math.Min(1.0, value));
}

View File

@@ -72,6 +72,8 @@ public sealed class ReachabilityUnionWriter
Lang = Trim(n.Lang) ?? string.Empty,
Kind = Trim(n.Kind) ?? string.Empty,
Display = Trim(n.Display),
CodeBlockHash = Trim(n.CodeBlockHash),
Symbol = n.Symbol?.Trimmed(),
Source = n.Source?.Trimmed(),
Attributes = (n.Attributes ?? ImmutableDictionary<string, string>.Empty)
.Where(kv => !string.IsNullOrWhiteSpace(kv.Key) && kv.Value is not null)
@@ -173,6 +175,17 @@ public sealed class ReachabilityUnionWriter
jw.WriteString("purl", node.Purl);
}
if (!string.IsNullOrWhiteSpace(node.CodeBlockHash))
{
jw.WriteString("code_block_hash", node.CodeBlockHash);
}
if (node.Symbol is not null)
{
jw.WritePropertyName("symbol");
WriteSymbol(jw, node.Symbol);
}
if (!string.IsNullOrWhiteSpace(node.SymbolDigest))
{
jw.WriteString("symbol_digest", node.SymbolDigest);
@@ -309,6 +322,31 @@ public sealed class ReachabilityUnionWriter
await writer.WriteAsync(Encoding.UTF8.GetString(json.ToArray())).ConfigureAwait(false);
}
private static void WriteSymbol(Utf8JsonWriter jw, ReachabilitySymbol symbol)
{
jw.WriteStartObject();
if (!string.IsNullOrWhiteSpace(symbol.Mangled))
{
jw.WriteString("mangled", symbol.Mangled);
}
if (!string.IsNullOrWhiteSpace(symbol.Demangled))
{
jw.WriteString("demangled", symbol.Demangled);
}
if (!string.IsNullOrWhiteSpace(symbol.Source))
{
jw.WriteString("source", symbol.Source);
}
if (symbol.Confidence is not null)
{
jw.WriteNumber("confidence", symbol.Confidence.Value);
}
jw.WriteEndObject();
}
private static void WriteSource(Utf8JsonWriter jw, ReachabilitySource source)
{
jw.WriteStartObject();
@@ -390,6 +428,8 @@ public sealed record ReachabilityUnionNode(
string Lang,
string Kind,
string? Display = null,
string? CodeBlockHash = null,
ReachabilitySymbol? Symbol = null,
ReachabilitySource? Source = null,
IReadOnlyDictionary<string, string>? Attributes = null,
string? Purl = null,

View File

@@ -52,7 +52,9 @@ public sealed record RichGraphNode(
string? BuildId,
IReadOnlyList<string>? Evidence,
IReadOnlyDictionary<string, string>? Attributes,
string? SymbolDigest)
string? SymbolDigest,
ReachabilitySymbol? Symbol = null,
string? CodeBlockHash = null)
{
public RichGraphNode Trimmed()
{
@@ -66,7 +68,9 @@ public sealed record RichGraphNode(
Kind = Kind.Trim(),
Display = string.IsNullOrWhiteSpace(Display) ? null : Display.Trim(),
BuildId = string.IsNullOrWhiteSpace(BuildId) ? null : BuildId.Trim(),
CodeBlockHash = string.IsNullOrWhiteSpace(CodeBlockHash) ? null : CodeBlockHash.Trim(),
SymbolDigest = string.IsNullOrWhiteSpace(SymbolDigest) ? null : SymbolDigest.Trim(),
Symbol = Symbol?.Trimmed(),
Evidence = Evidence is null
? Array.Empty<string>()
: Evidence.Where(e => !string.IsNullOrWhiteSpace(e)).Select(e => e.Trim()).OrderBy(e => e, StringComparer.Ordinal).ToArray(),
@@ -155,6 +159,21 @@ public static class RichGraphBuilder
var symbolDigest = ComputeSymbolDigest(symbolId);
nodeDigests[symbolId] = symbolDigest;
var codeBlockHash = node.CodeBlockHash;
if (string.IsNullOrWhiteSpace(codeBlockHash) && node.Attributes is not null && node.Attributes.TryGetValue("code_block_hash", out var cbh))
{
codeBlockHash = cbh;
}
var symbol = node.Symbol;
if (symbol is null && !string.IsNullOrWhiteSpace(node.Display))
{
var symbolSource = node.Attributes is not null && node.Attributes.TryGetValue("symbol_source", out var sourceHint)
? sourceHint
: null;
symbol = new ReachabilitySymbol(null, node.Display, symbolSource, null);
}
var codeId = node.Attributes is not null && node.Attributes.TryGetValue("code_id", out var cid)
? cid
: CodeId.FromSymbolId(symbolId);
@@ -170,7 +189,9 @@ public static class RichGraphBuilder
BuildId: node.Attributes is not null && node.Attributes.TryGetValue("build_id", out var bid) ? bid : null,
Evidence: node.Source?.Evidence is null ? Array.Empty<string>() : new[] { node.Source.Evidence },
Attributes: node.Attributes,
SymbolDigest: symbolDigest));
SymbolDigest: symbolDigest,
Symbol: symbol?.Trimmed(),
CodeBlockHash: string.IsNullOrWhiteSpace(codeBlockHash) ? null : codeBlockHash.Trim()));
}
var edges = new List<RichGraphEdge>();

View File

@@ -110,7 +110,13 @@ public sealed class RichGraphWriter
if (!string.IsNullOrWhiteSpace(node.CodeId)) writer.WriteString("code_id", node.CodeId);
if (!string.IsNullOrWhiteSpace(node.Purl)) writer.WriteString("purl", node.Purl);
if (!string.IsNullOrWhiteSpace(node.BuildId)) writer.WriteString("build_id", node.BuildId);
if (!string.IsNullOrWhiteSpace(node.CodeBlockHash)) writer.WriteString("code_block_hash", node.CodeBlockHash);
if (!string.IsNullOrWhiteSpace(node.SymbolDigest)) writer.WriteString("symbol_digest", node.SymbolDigest);
if (node.Symbol is not null)
{
writer.WritePropertyName("symbol");
WriteSymbol(writer, node.Symbol);
}
if (node.Evidence is { Count: > 0 })
{
@@ -182,6 +188,30 @@ public sealed class RichGraphWriter
writer.WriteEndObject();
}
private static void WriteSymbol(Utf8JsonWriter writer, ReachabilitySymbol symbol)
{
writer.WriteStartObject();
if (!string.IsNullOrWhiteSpace(symbol.Mangled))
{
writer.WriteString("mangled", symbol.Mangled);
}
if (!string.IsNullOrWhiteSpace(symbol.Demangled))
{
writer.WriteString("demangled", symbol.Demangled);
}
if (!string.IsNullOrWhiteSpace(symbol.Source))
{
writer.WriteString("source", symbol.Source);
}
if (symbol.Confidence is not null)
{
writer.WriteNumber("confidence", symbol.Confidence.Value);
}
writer.WriteEndObject();
}
}
public sealed record RichGraphWriteResult(

View File

@@ -1,19 +1,19 @@
# AGENTS
## Role
Provide durable catalog and artifact storage for the Scanner plane, spanning Mongo catalog collections and MinIO object storage. Expose repositories and services used by WebService and Worker components to persist job state, image metadata, and exported artefacts deterministically.
Provide durable catalog and artifact storage for the Scanner plane, spanning PostgreSQL catalog tables and MinIO/object storage. Expose repositories and services used by WebService and Worker components to persist job state, image metadata, and exported artefacts deterministically.
## Scope
- Mongo collections: artifacts, images, layers, links, jobs, lifecycle_rules, migrations.
- Metadata documents: enforce majority write/read concerns, UTC timestamps, deterministic identifiers (SHA-256 digests, ULIDs for jobs).
- Bootstrapper: create collections + indexes (unique digests, compound references, TTL on lifecycle rules, sparse lookup helpers) and run schema migrations.
- PostgreSQL schema: artifacts, images, layers, links, jobs, lifecycle_rules, runtime_events, entry_trace, ruby_packages, bun_packages.
- Metadata rows: enforce UTC timestamps, deterministic identifiers (SHA-256 digests, ULIDs for jobs).
- Migrations: SQL migrations executed via `AddStartupMigrations` to create schema and indexes (unique digests, compound references, TTL equivalents via timestamp columns).
- Object storage (MinIO/S3): manage bucket layout (layers/, images/, indexes/, attest/), immutability policies, deterministic paths, and retention classes.
- Services: coordinate dual-write between Mongo metadata and MinIO blobs, compute digests, manage reference counts, and expose typed repositories for WebService/Worker interactions.
- Services: coordinate dual-write between Postgres metadata and MinIO blobs, compute digests, manage reference counts, and expose typed repositories for WebService/Worker interactions.
## Participants
- Scanner.WebService binds configuration, runs bootstrapper during startup, and uses repositories to enqueue scans, look up catalog entries, and manage lifecycle policies.
- Scanner.WebService binds configuration, runs migrations during startup, and uses repositories to enqueue scans, look up catalog entries, and manage lifecycle policies.
- Scanner.Worker writes job progress, uploads SBOM artefacts, and updates artefact reference counts.
- Policy / Notify consumers resolve artefact metadata for reports via catalog APIs once exposed.
## Interfaces & contracts
- Options configured via `ScannerStorageOptions` (Mongo + object store). `EnsureValid` rejects incomplete/unsafe configuration.
- Mongo access uses `IMongoDatabase` scoped with majority `ReadConcern`/`WriteConcern` and cancellation tokens.
- Options configured via `ScannerStorageOptions` (Postgres + object store). `EnsureValid` rejects incomplete/unsafe configuration.
- Postgres access uses `ScannerDataSource` scoped sessions with UTC timezone and statement timeout; schema defaults to `scanner`.
- Object store abstraction (`IArtifactObjectStore`) encapsulates MinIO (S3) operations with server-side checksum validation and optional object-lock retain-until.
- Service APIs follow deterministic naming: digests normalized (`sha256:<hex>`), ULIDs sortable, timestamps ISO-8601 UTC.
## In/Out of scope
@@ -23,9 +23,9 @@ Out: HTTP endpoints, queue processing, analyzer logic, SBOM composition, policy
- Emit structured logs for catalog/object-store writes including correlation IDs and digests.
- Guard against double writes; idempotent operations keyed by digests.
- Do not log credentials; redact connection strings. Honour cancellation tokens.
- Metrics hooks (pending) must expose duration counters for Mongo and MinIO operations.
- Metrics hooks (pending) must expose duration counters for Postgres and MinIO operations.
## Tests
- Integration tests with ephemeral Mongo/MinIO stubs covering bootstrapper indexes, TTL enforcement, dual-write coordination, digest determinism, and majority read/write concerns.
- Integration tests with Testcontainers-backed Postgres and in-memory object store covering migrations, dual-write coordination, digest determinism, and Postgres session concerns.
## Required Reading
- `docs/modules/scanner/architecture.md`

View File

@@ -1,7 +1,5 @@
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Catalog;
namespace StellaOps.Scanner.Storage.Catalog;
public enum ArtifactDocumentType
{
LayerBom,
@@ -30,67 +28,50 @@ public enum ArtifactDocumentFormat
ObservationJson,
CompositionRecipeJson
}
[BsonIgnoreExtraElements]
public sealed class ArtifactDocument
{
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("type")]
public ArtifactDocumentType Type { get; set; }
= ArtifactDocumentType.ImageBom;
[BsonElement("format")]
public ArtifactDocumentFormat Format { get; set; }
= ArtifactDocumentFormat.CycloneDxJson;
[BsonElement("mediaType")]
public string MediaType { get; set; } = string.Empty;
[BsonElement("bytesSha256")]
public string BytesSha256 { get; set; } = string.Empty;
[BsonElement("sizeBytes")]
public long SizeBytes { get; set; }
= 0;
[BsonElement("immutable")]
public bool Immutable { get; set; }
= false;
[BsonElement("refCount")]
public long RefCount { get; set; }
= 0;
[BsonElement("rekor")]
[BsonIgnoreIfNull]
public RekorReference? Rekor { get; set; }
= null;
[BsonElement("createdAt")]
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("updatedAt")]
public DateTime UpdatedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("ttlClass")]
public string TtlClass { get; set; } = "default";
}
public sealed class RekorReference
{
[BsonElement("uuid")]
public string? Uuid { get; set; }
= null;
[BsonElement("index")]
public long? Index { get; set; }
= null;
[BsonElement("url")]
public string? Url { get; set; }
= null;
}
public sealed class ArtifactDocument
{
public string Id { get; set; } = string.Empty;
public ArtifactDocumentType Type { get; set; }
= ArtifactDocumentType.ImageBom;
public ArtifactDocumentFormat Format { get; set; }
= ArtifactDocumentFormat.CycloneDxJson;
public string MediaType { get; set; } = string.Empty;
public string BytesSha256 { get; set; } = string.Empty;
public long SizeBytes { get; set; }
= 0;
public bool Immutable { get; set; }
= false;
public long RefCount { get; set; }
= 0;
public RekorReference? Rekor { get; set; }
= null;
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
public DateTime UpdatedAtUtc { get; set; }
= DateTime.UtcNow;
public string TtlClass { get; set; } = "default";
}
public sealed class RekorReference
{
public string? Uuid { get; set; }
= null;
public long? Index { get; set; }
= null;
public string? Url { get; set; }
= null;
}

View File

@@ -1,79 +1,51 @@
using MongoDB.Bson.Serialization.Attributes;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Storage.Catalog;
[BsonIgnoreExtraElements]
public sealed class BunPackageInventoryDocument
{
[BsonId]
public string ScanId { get; set; } = string.Empty;
[BsonElement("imageDigest")]
[BsonIgnoreIfNull]
public string? ImageDigest { get; set; }
= null;
[BsonElement("generatedAtUtc")]
public DateTime GeneratedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("packages")]
public List<BunPackageDocument> Packages { get; set; }
= new();
}
[BsonIgnoreExtraElements]
public sealed class BunPackageDocument
{
[BsonElement("id")]
public string Id { get; set; } = string.Empty;
[BsonElement("name")]
public string Name { get; set; } = string.Empty;
[BsonElement("version")]
[BsonIgnoreIfNull]
public string? Version { get; set; }
= null;
[BsonElement("source")]
[BsonIgnoreIfNull]
public string? Source { get; set; }
= null;
[BsonElement("resolved")]
[BsonIgnoreIfNull]
public string? Resolved { get; set; }
= null;
[BsonElement("integrity")]
[BsonIgnoreIfNull]
public string? Integrity { get; set; }
= null;
[BsonElement("isDev")]
[BsonIgnoreIfNull]
public bool? IsDev { get; set; }
= null;
[BsonElement("isDirect")]
[BsonIgnoreIfNull]
public bool? IsDirect { get; set; }
= null;
[BsonElement("isPatched")]
[BsonIgnoreIfNull]
public bool? IsPatched { get; set; }
= null;
[BsonElement("provenance")]
[BsonIgnoreIfNull]
public BunPackageProvenance? Provenance { get; set; }
= null;
[BsonElement("metadata")]
[BsonIgnoreIfNull]
public Dictionary<string, string?>? Metadata { get; set; }
= null;
}

View File

@@ -1,23 +1,15 @@
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Catalog;
[BsonIgnoreExtraElements]
public sealed class EntryTraceDocument
{
[BsonId]
public string ScanId { get; set; } = string.Empty;
[BsonElement("image_digest")]
public string ImageDigest { get; set; } = string.Empty;
[BsonElement("generated_at")]
public DateTime GeneratedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("graph_json")]
public string GraphJson { get; set; } = string.Empty;
[BsonElement("ndjson")]
public List<string> Ndjson { get; set; } = new();
}

View File

@@ -1,29 +1,19 @@
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Catalog;
[BsonIgnoreExtraElements]
public sealed class ImageDocument
{
[BsonId]
public string ImageDigest { get; set; } = string.Empty;
[BsonElement("repository")]
public string Repository { get; set; } = string.Empty;
[BsonElement("tag")]
[BsonIgnoreIfNull]
public string? Tag { get; set; }
= null;
[BsonElement("architecture")]
public string Architecture { get; set; } = string.Empty;
[BsonElement("createdAt")]
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("lastSeenAt")]
public DateTime LastSeenAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,6 +1,3 @@
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Catalog;
public enum JobState
@@ -12,43 +9,29 @@ public enum JobState
Cancelled,
}
[BsonIgnoreExtraElements]
public sealed class JobDocument
{
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("kind")]
public string Kind { get; set; } = string.Empty;
[BsonElement("state")]
public JobState State { get; set; } = JobState.Pending;
[BsonElement("args")]
public BsonDocument Arguments { get; set; }
= new();
public string ArgumentsJson { get; set; }
= "{}";
[BsonElement("createdAt")]
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("startedAt")]
[BsonIgnoreIfNull]
public DateTime? StartedAtUtc { get; set; }
= null;
[BsonElement("completedAt")]
[BsonIgnoreIfNull]
public DateTime? CompletedAtUtc { get; set; }
= null;
[BsonElement("heartbeatAt")]
[BsonIgnoreIfNull]
public DateTime? HeartbeatAtUtc { get; set; }
= null;
[BsonElement("error")]
[BsonIgnoreIfNull]
public string? Error { get; set; }
= null;
}

View File

@@ -1,25 +1,17 @@
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Catalog;
[BsonIgnoreExtraElements]
public sealed class LayerDocument
{
[BsonId]
public string LayerDigest { get; set; } = string.Empty;
[BsonElement("mediaType")]
public string MediaType { get; set; } = string.Empty;
[BsonElement("sizeBytes")]
public long SizeBytes { get; set; }
= 0;
[BsonElement("createdAt")]
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("lastSeenAt")]
public DateTime LastSeenAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,25 +1,16 @@
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Catalog;
[BsonIgnoreExtraElements]
public sealed class LifecycleRuleDocument
{
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("artifactId")]
public string ArtifactId { get; set; } = string.Empty;
[BsonElement("class")]
public string Class { get; set; } = "default";
[BsonElement("expiresAt")]
[BsonIgnoreIfNull]
public DateTime? ExpiresAtUtc { get; set; }
= null;
[BsonElement("createdAt")]
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,5 +1,3 @@
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Catalog;
public enum LinkSourceType
@@ -8,23 +6,17 @@ public enum LinkSourceType
Layer,
}
[BsonIgnoreExtraElements]
public sealed class LinkDocument
{
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("fromType")]
public LinkSourceType FromType { get; set; }
= LinkSourceType.Image;
[BsonElement("fromDigest")]
public string FromDigest { get; set; } = string.Empty;
[BsonElement("artifactId")]
public string ArtifactId { get; set; } = string.Empty;
[BsonElement("createdAt")]
public DateTime CreatedAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,79 +1,51 @@
using MongoDB.Bson.Serialization.Attributes;
using StellaOps.Scanner.Core.Contracts;
namespace StellaOps.Scanner.Storage.Catalog;
[BsonIgnoreExtraElements]
public sealed class RubyPackageInventoryDocument
{
[BsonId]
public string ScanId { get; set; } = string.Empty;
[BsonElement("imageDigest")]
[BsonIgnoreIfNull]
public string? ImageDigest { get; set; }
= null;
[BsonElement("generatedAtUtc")]
public DateTime GeneratedAtUtc { get; set; }
= DateTime.UtcNow;
[BsonElement("packages")]
public List<RubyPackageDocument> Packages { get; set; }
= new();
}
[BsonIgnoreExtraElements]
public sealed class RubyPackageDocument
{
[BsonElement("id")]
public string Id { get; set; } = string.Empty;
[BsonElement("name")]
public string Name { get; set; } = string.Empty;
[BsonElement("version")]
[BsonIgnoreIfNull]
public string? Version { get; set; }
= null;
[BsonElement("source")]
[BsonIgnoreIfNull]
public string? Source { get; set; }
= null;
[BsonElement("platform")]
[BsonIgnoreIfNull]
public string? Platform { get; set; }
= null;
[BsonElement("groups")]
[BsonIgnoreIfNull]
public List<string>? Groups { get; set; }
= null;
[BsonElement("declaredOnly")]
[BsonIgnoreIfNull]
public bool? DeclaredOnly { get; set; }
= null;
[BsonElement("runtimeUsed")]
[BsonIgnoreIfNull]
public bool? RuntimeUsed { get; set; }
= null;
[BsonElement("provenance")]
[BsonIgnoreIfNull]
public RubyPackageProvenance? Provenance { get; set; }
= null;
[BsonElement("runtime")]
[BsonIgnoreIfNull]
public RubyPackageRuntime? Runtime { get; set; }
= null;
[BsonElement("metadata")]
[BsonIgnoreIfNull]
public Dictionary<string, string?>? Metadata { get; set; }
= null;
}

View File

@@ -1,89 +1,53 @@
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Catalog;
/// <summary>
/// MongoDB persistence model for runtime events emitted by the Zastava observer.
/// Persistence model for runtime events emitted by the Zastava observer.
/// </summary>
public sealed class RuntimeEventDocument
{
[BsonId]
[BsonRepresentation(BsonType.ObjectId)]
public string? Id { get; set; }
[BsonElement("eventId")]
[BsonRequired]
public string EventId { get; set; } = string.Empty;
[BsonElement("schemaVersion")]
[BsonRequired]
public string SchemaVersion { get; set; } = string.Empty;
[BsonElement("tenant")]
[BsonRequired]
public string Tenant { get; set; } = string.Empty;
[BsonElement("node")]
[BsonRequired]
public string Node { get; set; } = string.Empty;
[BsonElement("kind")]
[BsonRepresentation(BsonType.String)]
[BsonRequired]
public string Kind { get; set; } = string.Empty;
[BsonElement("when")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
public DateTime When { get; set; }
[BsonElement("receivedAt")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
public DateTime ReceivedAt { get; set; }
[BsonElement("expiresAt")]
[BsonDateTimeOptions(Kind = DateTimeKind.Utc)]
public DateTime ExpiresAt { get; set; }
[BsonElement("platform")]
public string? Platform { get; set; }
[BsonElement("namespace")]
public string? Namespace { get; set; }
[BsonElement("pod")]
public string? Pod { get; set; }
[BsonElement("container")]
public string? Container { get; set; }
[BsonElement("containerId")]
public string? ContainerId { get; set; }
[BsonElement("imageRef")]
public string? ImageRef { get; set; }
[BsonElement("imageDigest")]
public string? ImageDigest { get; set; }
[BsonElement("engine")]
public string? Engine { get; set; }
[BsonElement("engineVersion")]
public string? EngineVersion { get; set; }
[BsonElement("baselineDigest")]
public string? BaselineDigest { get; set; }
[BsonElement("imageSigned")]
public bool? ImageSigned { get; set; }
[BsonElement("sbomReferrer")]
public string? SbomReferrer { get; set; }
[BsonElement("buildId")]
public string? BuildId { get; set; }
[BsonElement("payload")]
public BsonDocument Payload { get; set; } = new();
public string PayloadJson { get; set; } = "{}";
}

View File

@@ -1,19 +1,18 @@
using System;
using System.Net.Http;
using Amazon;
using Amazon.S3;
using Amazon.Runtime;
using Amazon.S3;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Infrastructure.Postgres.Migrations;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.Storage.Migrations;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.ObjectStore;
using StellaOps.Scanner.Storage.Postgres;
using StellaOps.Scanner.Storage.Repositories;
using StellaOps.Scanner.Storage.Services;
@@ -25,7 +24,11 @@ public static class ServiceCollectionExtensions
{
ArgumentNullException.ThrowIfNull(configure);
services.AddOptions<ScannerStorageOptions>().Configure(configure).PostConfigure(options => options.EnsureValid());
services.AddOptions<ScannerStorageOptions>()
.Configure(configure)
.PostConfigure(options => options.EnsureValid())
.ValidateOnStart();
RegisterScannerStorageServices(services);
return services;
}
@@ -36,7 +39,8 @@ public static class ServiceCollectionExtensions
services.AddOptions<ScannerStorageOptions>()
.Bind(configuration)
.PostConfigure(options => options.EnsureValid());
.PostConfigure(options => options.EnsureValid())
.ValidateOnStart();
RegisterScannerStorageServices(services);
return services;
@@ -45,31 +49,30 @@ public static class ServiceCollectionExtensions
private static void RegisterScannerStorageServices(IServiceCollection services)
{
services.TryAddSingleton<TimeProvider>(TimeProvider.System);
services.TryAddSingleton(CreateMongoClient);
services.TryAddSingleton(CreateMongoDatabase);
services.TryAddSingleton<MongoCollectionProvider>();
services.TryAddEnumerable(ServiceDescriptor.Singleton<IMongoMigration, EnsureLifecycleRuleTtlMigration>());
services.TryAddSingleton(provider =>
{
var migrations = provider.GetServices<IMongoMigration>();
return new MongoMigrationRunner(
provider.GetRequiredService<IMongoDatabase>(),
migrations,
provider.GetRequiredService<ILogger<MongoMigrationRunner>>(),
TimeProvider.System);
});
services.TryAddSingleton<ScannerDataSource>();
services.TryAddSingleton<MongoBootstrapper>();
services.TryAddSingleton<ArtifactRepository>();
services.TryAddSingleton<ImageRepository>();
services.TryAddSingleton<LayerRepository>();
services.TryAddSingleton<LinkRepository>();
services.TryAddSingleton<JobRepository>();
services.TryAddSingleton<LifecycleRuleRepository>();
services.TryAddSingleton<RuntimeEventRepository>();
services.TryAddSingleton<EntryTraceRepository>();
services.TryAddSingleton<RubyPackageInventoryRepository>();
services.TryAddSingleton<BunPackageInventoryRepository>();
services.AddStartupMigrations<ScannerStorageOptions>(
ScannerDataSource.DefaultSchema,
"Scanner.Storage",
typeof(ScannerDataSource).Assembly,
options => options.Postgres.ConnectionString);
services.AddMigrationStatus<ScannerStorageOptions>(
ScannerDataSource.DefaultSchema,
"Scanner.Storage",
typeof(ScannerDataSource).Assembly,
options => options.Postgres.ConnectionString);
services.AddScoped<ArtifactRepository>();
services.AddScoped<ImageRepository>();
services.AddScoped<LayerRepository>();
services.AddScoped<LinkRepository>();
services.AddScoped<JobRepository>();
services.AddScoped<LifecycleRuleRepository>();
services.AddScoped<RuntimeEventRepository>();
services.AddScoped<EntryTraceRepository>();
services.AddScoped<RubyPackageInventoryRepository>();
services.AddScoped<BunPackageInventoryRepository>();
services.AddSingleton<IEntryTraceResultStore, EntryTraceResultStore>();
services.AddSingleton<IRubyPackageInventoryStore, RubyPackageInventoryStore>();
services.AddSingleton<IBunPackageInventoryStore, BunPackageInventoryStore>();
@@ -124,33 +127,6 @@ public static class ServiceCollectionExtensions
services.TryAddSingleton<ArtifactStorageService>();
}
private static IMongoClient CreateMongoClient(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value;
options.EnsureValid();
var settings = MongoClientSettings.FromConnectionString(options.Mongo.ConnectionString);
settings.RetryReads = true;
settings.RetryWrites = true;
settings.DirectConnection = false;
settings.ReadPreference = ReadPreference.PrimaryPreferred;
settings.ServerSelectionTimeout = options.Mongo.CommandTimeout;
settings.ConnectTimeout = options.Mongo.CommandTimeout;
settings.SocketTimeout = options.Mongo.CommandTimeout;
settings.ReadConcern = options.Mongo.UseMajorityReadConcern ? ReadConcern.Majority : ReadConcern.Local;
settings.WriteConcern = options.Mongo.UseMajorityWriteConcern ? WriteConcern.WMajority : WriteConcern.W1;
return new MongoClient(settings);
}
private static IMongoDatabase CreateMongoDatabase(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value;
var client = provider.GetRequiredService<IMongoClient>();
var databaseName = options.Mongo.ResolveDatabaseName();
return client.GetDatabase(databaseName);
}
private static IAmazonS3 CreateAmazonS3Client(IServiceProvider provider)
{
var options = provider.GetRequiredService<IOptions<ScannerStorageOptions>>().Value.ObjectStore;

View File

@@ -1,30 +0,0 @@
using System.Linq;
using MongoDB.Driver;
using StellaOps.Scanner.Storage.Catalog;
namespace StellaOps.Scanner.Storage.Migrations;
public sealed class EnsureLifecycleRuleTtlMigration : IMongoMigration
{
public string Id => "20251018-lifecycle-ttl";
public string Description => "Ensure lifecycle_rules expiresAt TTL index exists.";
public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
var collection = database.GetCollection<LifecycleRuleDocument>(ScannerStorageDefaults.Collections.LifecycleRules);
var indexes = await collection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false);
var existing = await indexes.ToListAsync(cancellationToken).ConfigureAwait(false);
if (existing.Any(x => string.Equals(x["name"].AsString, "lifecycle_expiresAt", StringComparison.Ordinal)))
{
return;
}
var model = new CreateIndexModel<LifecycleRuleDocument>(
Builders<LifecycleRuleDocument>.IndexKeys.Ascending(x => x.ExpiresAtUtc),
new CreateIndexOptions { Name = "lifecycle_expiresAt", ExpireAfter = TimeSpan.Zero });
await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -1,12 +0,0 @@
using MongoDB.Driver;
namespace StellaOps.Scanner.Storage.Migrations;
public interface IMongoMigration
{
string Id { get; }
string Description { get; }
Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken);
}

View File

@@ -1,19 +0,0 @@
using MongoDB.Bson.Serialization.Attributes;
namespace StellaOps.Scanner.Storage.Migrations;
[BsonIgnoreExtraElements]
internal sealed class MongoMigrationDocument
{
[BsonId]
public string Id { get; set; } = string.Empty;
[BsonElement("description")]
[BsonIgnoreIfNull]
public string? Description { get; set; }
= null;
[BsonElement("appliedAt")]
public DateTime AppliedAtUtc { get; set; }
= DateTime.UtcNow;
}

View File

@@ -1,94 +0,0 @@
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
namespace StellaOps.Scanner.Storage.Migrations;
public sealed class MongoMigrationRunner
{
private readonly IMongoDatabase _database;
private readonly IReadOnlyList<IMongoMigration> _migrations;
private readonly ILogger<MongoMigrationRunner> _logger;
private readonly TimeProvider _timeProvider;
public MongoMigrationRunner(
IMongoDatabase database,
IEnumerable<IMongoMigration> migrations,
ILogger<MongoMigrationRunner> logger,
TimeProvider? timeProvider = null)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_migrations = (migrations ?? throw new ArgumentNullException(nameof(migrations)))
.OrderBy(m => m.Id, StringComparer.Ordinal)
.ToArray();
}
public async Task RunAsync(CancellationToken cancellationToken)
{
if (_migrations.Count == 0)
{
return;
}
await EnsureCollectionExistsAsync(_database, cancellationToken).ConfigureAwait(false);
var collection = _database.GetCollection<MongoMigrationDocument>(ScannerStorageDefaults.Collections.Migrations);
var applied = await LoadAppliedMigrationIdsAsync(collection, cancellationToken).ConfigureAwait(false);
foreach (var migration in _migrations)
{
if (applied.Contains(migration.Id, StringComparer.Ordinal))
{
continue;
}
_logger.LogInformation("Applying scanner Mongo migration {MigrationId}: {Description}", migration.Id, migration.Description);
try
{
await migration.ApplyAsync(_database, cancellationToken).ConfigureAwait(false);
var document = new MongoMigrationDocument
{
Id = migration.Id,
Description = string.IsNullOrWhiteSpace(migration.Description) ? null : migration.Description,
AppliedAtUtc = _timeProvider.GetUtcNow().UtcDateTime,
};
await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Scanner Mongo migration {MigrationId} applied", migration.Id);
}
catch (Exception ex)
{
_logger.LogError(ex, "Scanner Mongo migration {MigrationId} failed", migration.Id);
throw;
}
}
}
private static async Task EnsureCollectionExistsAsync(IMongoDatabase database, CancellationToken cancellationToken)
{
using var cursor = await database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
var names = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false);
if (!names.Contains(ScannerStorageDefaults.Collections.Migrations, StringComparer.Ordinal))
{
await database.CreateCollectionAsync(ScannerStorageDefaults.Collections.Migrations, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
private static async Task<HashSet<string>> LoadAppliedMigrationIdsAsync(
IMongoCollection<MongoMigrationDocument> collection,
CancellationToken cancellationToken)
{
using var cursor = await collection.FindAsync(FilterDefinition<MongoMigrationDocument>.Empty, cancellationToken: cancellationToken).ConfigureAwait(false);
var documents = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false);
var ids = new HashSet<string>(StringComparer.Ordinal);
foreach (var doc in documents)
{
if (!string.IsNullOrWhiteSpace(doc.Id))
{
ids.Add(doc.Id);
}
}
return ids;
}
}

View File

@@ -1,237 +0,0 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Migrations;
namespace StellaOps.Scanner.Storage.Mongo;
public sealed class MongoBootstrapper
{
private readonly IMongoDatabase _database;
private readonly ScannerStorageOptions _options;
private readonly ILogger<MongoBootstrapper> _logger;
private readonly MongoMigrationRunner _migrationRunner;
public MongoBootstrapper(
IMongoDatabase database,
IOptions<ScannerStorageOptions> options,
ILogger<MongoBootstrapper> logger,
MongoMigrationRunner migrationRunner)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_migrationRunner = migrationRunner ?? throw new ArgumentNullException(nameof(migrationRunner));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
}
public async Task InitializeAsync(CancellationToken cancellationToken)
{
_options.EnsureValid();
await EnsureCollectionsAsync(cancellationToken).ConfigureAwait(false);
await EnsureIndexesAsync(cancellationToken).ConfigureAwait(false);
await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false);
}
private async Task EnsureCollectionsAsync(CancellationToken cancellationToken)
{
var targetCollections = new[]
{
ScannerStorageDefaults.Collections.Artifacts,
ScannerStorageDefaults.Collections.Images,
ScannerStorageDefaults.Collections.Layers,
ScannerStorageDefaults.Collections.Links,
ScannerStorageDefaults.Collections.Jobs,
ScannerStorageDefaults.Collections.LifecycleRules,
ScannerStorageDefaults.Collections.RuntimeEvents,
ScannerStorageDefaults.Collections.RubyPackages,
ScannerStorageDefaults.Collections.Migrations,
};
using var cursor = await _database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
var existing = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false);
foreach (var name in targetCollections)
{
if (existing.Contains(name, StringComparer.Ordinal))
{
continue;
}
_logger.LogInformation("Creating Mongo collection {Collection}", name);
await _database.CreateCollectionAsync(name, cancellationToken: cancellationToken).ConfigureAwait(false);
}
}
private async Task EnsureIndexesAsync(CancellationToken cancellationToken)
{
await EnsureArtifactIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureImageIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureLayerIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureLinkIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureJobIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureLifecycleIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureRuntimeEventIndexesAsync(cancellationToken).ConfigureAwait(false);
await EnsureRubyPackageIndexesAsync(cancellationToken).ConfigureAwait(false);
}
private Task EnsureArtifactIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<ArtifactDocument>(ScannerStorageDefaults.Collections.Artifacts);
var models = new List<CreateIndexModel<ArtifactDocument>>
{
new(
Builders<ArtifactDocument>.IndexKeys
.Ascending(x => x.Type)
.Ascending(x => x.BytesSha256),
new CreateIndexOptions { Name = "artifact_type_bytesSha256", Unique = true }),
new(
Builders<ArtifactDocument>.IndexKeys.Ascending(x => x.RefCount),
new CreateIndexOptions { Name = "artifact_refCount" }),
new(
Builders<ArtifactDocument>.IndexKeys.Ascending(x => x.CreatedAtUtc),
new CreateIndexOptions { Name = "artifact_createdAt" })
};
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
private Task EnsureImageIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<ImageDocument>(ScannerStorageDefaults.Collections.Images);
var models = new List<CreateIndexModel<ImageDocument>>
{
new(
Builders<ImageDocument>.IndexKeys
.Ascending(x => x.Repository)
.Ascending(x => x.Tag),
new CreateIndexOptions { Name = "image_repo_tag" }),
new(
Builders<ImageDocument>.IndexKeys.Ascending(x => x.LastSeenAtUtc),
new CreateIndexOptions { Name = "image_lastSeen" })
};
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
private Task EnsureLayerIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<LayerDocument>(ScannerStorageDefaults.Collections.Layers);
var models = new List<CreateIndexModel<LayerDocument>>
{
new(
Builders<LayerDocument>.IndexKeys.Ascending(x => x.LastSeenAtUtc),
new CreateIndexOptions { Name = "layer_lastSeen" })
};
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
private Task EnsureLinkIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<LinkDocument>(ScannerStorageDefaults.Collections.Links);
var models = new List<CreateIndexModel<LinkDocument>>
{
new(
Builders<LinkDocument>.IndexKeys
.Ascending(x => x.FromType)
.Ascending(x => x.FromDigest)
.Ascending(x => x.ArtifactId),
new CreateIndexOptions { Name = "link_from_artifact", Unique = true })
};
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
private Task EnsureJobIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<JobDocument>(ScannerStorageDefaults.Collections.Jobs);
var models = new List<CreateIndexModel<JobDocument>>
{
new(
Builders<JobDocument>.IndexKeys
.Ascending(x => x.State)
.Ascending(x => x.CreatedAtUtc),
new CreateIndexOptions { Name = "job_state_createdAt" }),
new(
Builders<JobDocument>.IndexKeys.Ascending(x => x.HeartbeatAtUtc),
new CreateIndexOptions { Name = "job_heartbeat" })
};
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
private Task EnsureLifecycleIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<LifecycleRuleDocument>(ScannerStorageDefaults.Collections.LifecycleRules);
var expiresIndex = new CreateIndexModel<LifecycleRuleDocument>(
Builders<LifecycleRuleDocument>.IndexKeys.Ascending(x => x.ExpiresAtUtc),
new CreateIndexOptions
{
Name = "lifecycle_expiresAt",
ExpireAfter = TimeSpan.Zero,
});
var artifactIndex = new CreateIndexModel<LifecycleRuleDocument>(
Builders<LifecycleRuleDocument>.IndexKeys
.Ascending(x => x.ArtifactId)
.Ascending(x => x.Class),
new CreateIndexOptions { Name = "lifecycle_artifact_class", Unique = true });
return collection.Indexes.CreateManyAsync(new[] { expiresIndex, artifactIndex }, cancellationToken);
}
private Task EnsureRuntimeEventIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<RuntimeEventDocument>(ScannerStorageDefaults.Collections.RuntimeEvents);
var models = new List<CreateIndexModel<RuntimeEventDocument>>
{
new(
Builders<RuntimeEventDocument>.IndexKeys.Ascending(x => x.EventId),
new CreateIndexOptions { Name = "runtime_event_eventId", Unique = true }),
new(
Builders<RuntimeEventDocument>.IndexKeys
.Ascending(x => x.Tenant)
.Ascending(x => x.Node)
.Ascending(x => x.When),
new CreateIndexOptions { Name = "runtime_event_tenant_node_when" }),
new(
Builders<RuntimeEventDocument>.IndexKeys
.Ascending(x => x.ImageDigest)
.Descending(x => x.When),
new CreateIndexOptions { Name = "runtime_event_imageDigest_when" }),
new(
Builders<RuntimeEventDocument>.IndexKeys
.Ascending(x => x.BuildId)
.Descending(x => x.When),
new CreateIndexOptions { Name = "runtime_event_buildId_when" }),
new(
Builders<RuntimeEventDocument>.IndexKeys.Ascending(x => x.ExpiresAt),
new CreateIndexOptions
{
Name = "runtime_event_expiresAt",
ExpireAfter = TimeSpan.Zero
})
};
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
private Task EnsureRubyPackageIndexesAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<RubyPackageInventoryDocument>(ScannerStorageDefaults.Collections.RubyPackages);
var models = new List<CreateIndexModel<RubyPackageInventoryDocument>>
{
new(
Builders<RubyPackageInventoryDocument>.IndexKeys.Ascending(x => x.ImageDigest),
new CreateIndexOptions { Name = "rubyPackages_imageDigest", Sparse = true }),
new(
Builders<RubyPackageInventoryDocument>.IndexKeys.Ascending(x => x.GeneratedAtUtc),
new CreateIndexOptions { Name = "rubyPackages_generatedAt" })
};
return collection.Indexes.CreateManyAsync(models, cancellationToken);
}
}

View File

@@ -1,45 +0,0 @@
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Scanner.Storage.Catalog;
namespace StellaOps.Scanner.Storage.Mongo;
public sealed class MongoCollectionProvider
{
private readonly IMongoDatabase _database;
private readonly MongoOptions _options;
public MongoCollectionProvider(IMongoDatabase database, IOptions<ScannerStorageOptions> options)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value.Mongo;
}
public IMongoCollection<ArtifactDocument> Artifacts => GetCollection<ArtifactDocument>(ScannerStorageDefaults.Collections.Artifacts);
public IMongoCollection<ImageDocument> Images => GetCollection<ImageDocument>(ScannerStorageDefaults.Collections.Images);
public IMongoCollection<LayerDocument> Layers => GetCollection<LayerDocument>(ScannerStorageDefaults.Collections.Layers);
public IMongoCollection<LinkDocument> Links => GetCollection<LinkDocument>(ScannerStorageDefaults.Collections.Links);
public IMongoCollection<JobDocument> Jobs => GetCollection<JobDocument>(ScannerStorageDefaults.Collections.Jobs);
public IMongoCollection<LifecycleRuleDocument> LifecycleRules => GetCollection<LifecycleRuleDocument>(ScannerStorageDefaults.Collections.LifecycleRules);
public IMongoCollection<RuntimeEventDocument> RuntimeEvents => GetCollection<RuntimeEventDocument>(ScannerStorageDefaults.Collections.RuntimeEvents);
public IMongoCollection<EntryTraceDocument> EntryTrace => GetCollection<EntryTraceDocument>(ScannerStorageDefaults.Collections.EntryTrace);
public IMongoCollection<RubyPackageInventoryDocument> RubyPackages => GetCollection<RubyPackageInventoryDocument>(ScannerStorageDefaults.Collections.RubyPackages);
public IMongoCollection<BunPackageInventoryDocument> BunPackages => GetCollection<BunPackageInventoryDocument>(ScannerStorageDefaults.Collections.BunPackages);
private IMongoCollection<TDocument> GetCollection<TDocument>(string name)
{
var database = _database;
if (_options.UseMajorityReadConcern)
{
database = database.WithReadConcern(ReadConcern.Majority);
}
if (_options.UseMajorityWriteConcern)
{
database = database.WithWriteConcern(WriteConcern.WMajority);
}
return database.GetCollection<TDocument>(name);
}
}

View File

@@ -0,0 +1,128 @@
-- scanner storage baseline schema
-- schema: created externally via search_path; tables prefixed for clarity
CREATE TABLE IF NOT EXISTS artifacts (
id TEXT PRIMARY KEY,
type TEXT NOT NULL,
format TEXT NOT NULL,
media_type TEXT NOT NULL,
bytes_sha256 TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
immutable BOOLEAN NOT NULL DEFAULT FALSE,
ref_count BIGINT NOT NULL DEFAULT 0,
rekor JSONB,
ttl_class TEXT NOT NULL DEFAULT 'default',
created_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS ix_artifacts_type_bytes ON artifacts (type, bytes_sha256);
CREATE INDEX IF NOT EXISTS ix_artifacts_refcount ON artifacts (ref_count);
CREATE INDEX IF NOT EXISTS ix_artifacts_created_at ON artifacts (created_at_utc);
CREATE TABLE IF NOT EXISTS images (
image_digest TEXT PRIMARY KEY,
repository TEXT NOT NULL,
tag TEXT,
architecture TEXT,
created_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_seen_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS ix_images_repo_tag ON images (repository, tag);
CREATE INDEX IF NOT EXISTS ix_images_last_seen ON images (last_seen_at_utc);
CREATE TABLE IF NOT EXISTS layers (
layer_digest TEXT PRIMARY KEY,
media_type TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
created_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_seen_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS ix_layers_last_seen ON layers (last_seen_at_utc);
CREATE TABLE IF NOT EXISTS links (
id TEXT PRIMARY KEY,
from_type TEXT NOT NULL,
from_digest TEXT NOT NULL,
artifact_id TEXT NOT NULL REFERENCES artifacts(id) ON DELETE CASCADE,
created_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS ix_links_from_artifact ON links (from_type, from_digest, artifact_id);
CREATE TYPE job_state AS ENUM ('Pending','Running','Succeeded','Failed','Cancelled');
CREATE TABLE IF NOT EXISTS jobs (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
state job_state NOT NULL DEFAULT 'Pending',
args JSONB NOT NULL,
created_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW(),
started_at_utc TIMESTAMPTZ,
completed_at_utc TIMESTAMPTZ,
heartbeat_at_utc TIMESTAMPTZ,
error TEXT
);
CREATE INDEX IF NOT EXISTS ix_jobs_state_created ON jobs (state, created_at_utc);
CREATE INDEX IF NOT EXISTS ix_jobs_heartbeat ON jobs (heartbeat_at_utc);
CREATE TABLE IF NOT EXISTS lifecycle_rules (
id TEXT PRIMARY KEY,
artifact_id TEXT NOT NULL REFERENCES artifacts(id) ON DELETE CASCADE,
class TEXT NOT NULL,
expires_at_utc TIMESTAMPTZ,
created_at_utc TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS ix_lifecycle_artifact_class ON lifecycle_rules (artifact_id, class);
CREATE INDEX IF NOT EXISTS ix_lifecycle_expires ON lifecycle_rules (expires_at_utc);
CREATE TABLE IF NOT EXISTS runtime_events (
id TEXT PRIMARY KEY,
event_id TEXT NOT NULL,
schema_version TEXT NOT NULL,
tenant TEXT NOT NULL,
node TEXT NOT NULL,
kind TEXT NOT NULL,
"when" TIMESTAMPTZ NOT NULL,
received_at TIMESTAMPTZ NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
platform TEXT,
namespace TEXT,
pod TEXT,
container TEXT,
container_id TEXT,
image_ref TEXT,
image_digest TEXT,
engine TEXT,
engine_version TEXT,
baseline_digest TEXT,
image_signed BOOLEAN,
sbom_referrer TEXT,
build_id TEXT,
payload JSONB NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS ix_runtime_event_id ON runtime_events (event_id);
CREATE INDEX IF NOT EXISTS ix_runtime_image_digest_when ON runtime_events (image_digest, "when" DESC);
CREATE INDEX IF NOT EXISTS ix_runtime_build_id_when ON runtime_events (build_id, "when" DESC);
CREATE INDEX IF NOT EXISTS ix_runtime_expires_at ON runtime_events (expires_at);
CREATE TABLE IF NOT EXISTS entry_trace (
scan_id TEXT PRIMARY KEY,
image_digest TEXT NOT NULL,
generated_at_utc TIMESTAMPTZ NOT NULL,
graph_json TEXT NOT NULL,
ndjson TEXT[] NOT NULL
);
CREATE TABLE IF NOT EXISTS ruby_packages (
scan_id TEXT PRIMARY KEY,
image_digest TEXT,
generated_at_utc TIMESTAMPTZ NOT NULL,
packages JSONB NOT NULL
);
CREATE TABLE IF NOT EXISTS bun_packages (
scan_id TEXT PRIMARY KEY,
image_digest TEXT,
generated_at_utc TIMESTAMPTZ NOT NULL,
packages JSONB NOT NULL
);

View File

@@ -0,0 +1,6 @@
namespace StellaOps.Scanner.Storage.Postgres.Migrations;
internal static class MigrationIds
{
public const string CreateTables = "001_create_tables.sql";
}

View File

@@ -0,0 +1,41 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Infrastructure.Postgres.Connections;
using StellaOps.Infrastructure.Postgres.Options;
namespace StellaOps.Scanner.Storage.Postgres;
/// <summary>
/// Scanner-specific PostgreSQL data source with schema defaults.
/// </summary>
public sealed class ScannerDataSource : DataSourceBase
{
/// <summary>
/// Default schema for scanner storage tables.
/// </summary>
public const string DefaultSchema = ScannerStorageDefaults.DefaultSchemaName;
protected override string ModuleName => "Scanner.Storage";
public ScannerDataSource(IOptions<ScannerStorageOptions> options, ILogger<ScannerDataSource> logger)
: base(ApplyDefaults(options?.Value.Postgres), logger)
{
}
private static PostgresOptions ApplyDefaults(PostgresOptions? options)
{
ArgumentNullException.ThrowIfNull(options);
if (string.IsNullOrWhiteSpace(options.SchemaName))
{
options.SchemaName = DefaultSchema;
}
if (options.CommandTimeoutSeconds <= 0)
{
options.CommandTimeoutSeconds = 30;
}
return options;
}
}

View File

@@ -1,73 +1,176 @@
using MongoDB.Driver;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class ArtifactRepository
public sealed class ArtifactRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.artifacts";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
private readonly TimeProvider _timeProvider;
public ArtifactRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null)
public ArtifactRepository(
ScannerDataSource dataSource,
ILogger<ArtifactRepository> logger,
TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<ArtifactDocument?> GetAsync(string artifactId, CancellationToken cancellationToken)
public Task<ArtifactDocument?> GetAsync(string artifactId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
return await _collections.Artifacts
.Find(x => x.Id == artifactId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var sql = $"""
SELECT id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
rekor, ttl_class, created_at_utc, updated_at_utc
FROM {Table}
WHERE id = @id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "id", artifactId),
MapArtifact,
cancellationToken);
}
public async Task UpsertAsync(ArtifactDocument document, CancellationToken cancellationToken)
public Task UpsertAsync(ArtifactDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
document.UpdatedAtUtc = now;
var options = new ReplaceOptions { IsUpsert = true };
await _collections.Artifacts
.ReplaceOneAsync(x => x.Id == document.Id, document, options, cancellationToken)
.ConfigureAwait(false);
var sql = $"""
INSERT INTO {Table} (
id, type, format, media_type, bytes_sha256, size_bytes, immutable, ref_count,
rekor, ttl_class, created_at_utc, updated_at_utc
)
VALUES (
@id, @type, @format, @media_type, @bytes_sha256, @size_bytes, @immutable, @ref_count,
@rekor::jsonb, @ttl_class, @created_at_utc, @updated_at_utc
)
ON CONFLICT (id) DO UPDATE SET
type = EXCLUDED.type,
format = EXCLUDED.format,
media_type = EXCLUDED.media_type,
bytes_sha256 = EXCLUDED.bytes_sha256,
size_bytes = EXCLUDED.size_bytes,
immutable = EXCLUDED.immutable,
ref_count = EXCLUDED.ref_count,
rekor = EXCLUDED.rekor,
ttl_class = EXCLUDED.ttl_class,
updated_at_utc = EXCLUDED.updated_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "type", document.Type.ToString());
AddParameter(cmd, "format", document.Format.ToString());
AddParameter(cmd, "media_type", document.MediaType);
AddParameter(cmd, "bytes_sha256", document.BytesSha256);
AddParameter(cmd, "size_bytes", document.SizeBytes);
AddParameter(cmd, "immutable", document.Immutable);
AddParameter(cmd, "ref_count", document.RefCount);
AddJsonbParameter(cmd, "rekor", SerializeRekor(document.Rekor));
AddParameter(cmd, "ttl_class", document.TtlClass);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "updated_at_utc", document.UpdatedAtUtc);
},
cancellationToken);
}
public async Task UpdateRekorAsync(string artifactId, RekorReference reference, CancellationToken cancellationToken)
public Task UpdateRekorAsync(string artifactId, RekorReference reference, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
ArgumentNullException.ThrowIfNull(reference);
var now = _timeProvider.GetUtcNow().UtcDateTime;
var update = Builders<ArtifactDocument>.Update
.Set(x => x.Rekor, reference)
.Set(x => x.UpdatedAtUtc, now);
await _collections.Artifacts.UpdateOneAsync(x => x.Id == artifactId, update, cancellationToken: cancellationToken).ConfigureAwait(false);
var sql = $"""
UPDATE {Table}
SET rekor = @rekor::jsonb,
updated_at_utc = @updated_at_utc
WHERE id = @id
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", artifactId);
AddJsonbParameter(cmd, "rekor", SerializeRekor(reference));
AddParameter(cmd, "updated_at_utc", now);
},
cancellationToken);
}
public async Task<long> IncrementRefCountAsync(string artifactId, long delta, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(artifactId);
var now = _timeProvider.GetUtcNow().UtcDateTime;
var update = Builders<ArtifactDocument>.Update
.Inc(x => x.RefCount, delta)
.Set(x => x.UpdatedAtUtc, now);
var sql = $"""
UPDATE {Table}
SET ref_count = ref_count + @delta,
updated_at_utc = NOW() AT TIME ZONE 'UTC'
WHERE id = @id
RETURNING ref_count
""";
var options = new FindOneAndUpdateOptions<ArtifactDocument, ArtifactDocument>
{
ReturnDocument = ReturnDocument.After,
IsUpsert = false,
};
var result = await ExecuteScalarAsync<long?>(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", artifactId);
AddParameter(cmd, "delta", delta);
},
cancellationToken).ConfigureAwait(false);
var result = await _collections.Artifacts
.FindOneAndUpdateAsync<ArtifactDocument, ArtifactDocument>(x => x.Id == artifactId, update, options, cancellationToken)
.ConfigureAwait(false);
return result?.RefCount ?? 0;
return result ?? 0;
}
private static ArtifactDocument MapArtifact(NpgsqlDataReader reader)
{
var rekorOrdinal = reader.GetOrdinal("rekor");
return new ArtifactDocument
{
Id = reader.GetString(reader.GetOrdinal("id")),
Type = ParseEnum(reader.GetString(reader.GetOrdinal("type")), ArtifactDocumentType.ImageBom),
Format = ParseEnum(reader.GetString(reader.GetOrdinal("format")), ArtifactDocumentFormat.CycloneDxJson),
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
BytesSha256 = reader.GetString(reader.GetOrdinal("bytes_sha256")),
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
Immutable = reader.GetBoolean(reader.GetOrdinal("immutable")),
RefCount = reader.GetInt64(reader.GetOrdinal("ref_count")),
Rekor = reader.IsDBNull(rekorOrdinal)
? null
: JsonSerializer.Deserialize<RekorReference>(reader.GetString(rekorOrdinal), JsonOptions),
TtlClass = reader.GetString(reader.GetOrdinal("ttl_class")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
UpdatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("updated_at_utc")),
};
}
private static TEnum ParseEnum<TEnum>(string value, TEnum fallback) where TEnum : struct
=> Enum.TryParse<TEnum>(value, ignoreCase: true, out var parsed) ? parsed : fallback;
private static string? SerializeRekor(RekorReference? reference)
=> reference is null ? null : JsonSerializer.Serialize(reference, JsonOptions);
}

View File

@@ -1,33 +1,81 @@
using MongoDB.Driver;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class BunPackageInventoryRepository
public sealed class BunPackageInventoryRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.bun_packages";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
public BunPackageInventoryRepository(MongoCollectionProvider collections)
public BunPackageInventoryRepository(ScannerDataSource dataSource, ILogger<BunPackageInventoryRepository> logger)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
}
public async Task<BunPackageInventoryDocument?> GetAsync(string scanId, CancellationToken cancellationToken)
public Task<BunPackageInventoryDocument?> GetAsync(string scanId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
return await _collections.BunPackages
.Find(x => x.ScanId == scanId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var sql = $"""
SELECT scan_id, image_digest, generated_at_utc, packages
FROM {Table}
WHERE scan_id = @scan_id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "scan_id", scanId),
MapInventory,
cancellationToken);
}
public async Task UpsertAsync(BunPackageInventoryDocument document, CancellationToken cancellationToken)
public Task UpsertAsync(BunPackageInventoryDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var options = new ReplaceOptions { IsUpsert = true };
await _collections.BunPackages
.ReplaceOneAsync(x => x.ScanId == document.ScanId, document, options, cancellationToken)
.ConfigureAwait(false);
var sql = $"""
INSERT INTO {Table} (scan_id, image_digest, generated_at_utc, packages)
VALUES (@scan_id, @image_digest, @generated_at_utc, @packages::jsonb)
ON CONFLICT (scan_id) DO UPDATE SET
image_digest = EXCLUDED.image_digest,
generated_at_utc = EXCLUDED.generated_at_utc,
packages = EXCLUDED.packages
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "scan_id", document.ScanId);
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "generated_at_utc", document.GeneratedAtUtc);
AddJsonbParameter(cmd, "packages", JsonSerializer.Serialize(document.Packages ?? new List<BunPackageDocument>(), JsonOptions));
},
cancellationToken);
}
private static BunPackageInventoryDocument MapInventory(NpgsqlDataReader reader)
{
var packagesJson = reader.GetString(reader.GetOrdinal("packages"));
var packages = string.IsNullOrWhiteSpace(packagesJson)
? new List<BunPackageDocument>()
: JsonSerializer.Deserialize<List<BunPackageDocument>>(packagesJson, JsonOptions) ?? new List<BunPackageDocument>();
return new BunPackageInventoryDocument
{
ScanId = reader.GetString(reader.GetOrdinal("scan_id")),
ImageDigest = GetNullableString(reader, reader.GetOrdinal("image_digest")),
GeneratedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("generated_at_utc")),
Packages = packages
};
}
}

View File

@@ -1,33 +1,74 @@
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class EntryTraceRepository
public sealed class EntryTraceRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.entry_trace";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
public EntryTraceRepository(MongoCollectionProvider collections)
public EntryTraceRepository(ScannerDataSource dataSource, ILogger<EntryTraceRepository> logger)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
}
public async Task<EntryTraceDocument?> GetAsync(string scanId, CancellationToken cancellationToken)
public Task<EntryTraceDocument?> GetAsync(string scanId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
return await _collections.EntryTrace
.Find(x => x.ScanId == scanId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var sql = $"""
SELECT scan_id, image_digest, generated_at_utc, graph_json, ndjson
FROM {Table}
WHERE scan_id = @scan_id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "scan_id", scanId),
MapEntryTrace,
cancellationToken);
}
public async Task UpsertAsync(EntryTraceDocument document, CancellationToken cancellationToken)
public Task UpsertAsync(EntryTraceDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var options = new ReplaceOptions { IsUpsert = true };
await _collections.EntryTrace
.ReplaceOneAsync(x => x.ScanId == document.ScanId, document, options, cancellationToken)
.ConfigureAwait(false);
var sql = $"""
INSERT INTO {Table} (scan_id, image_digest, generated_at_utc, graph_json, ndjson)
VALUES (@scan_id, @image_digest, @generated_at_utc, @graph_json, @ndjson)
ON CONFLICT (scan_id) DO UPDATE SET
image_digest = EXCLUDED.image_digest,
generated_at_utc = EXCLUDED.generated_at_utc,
graph_json = EXCLUDED.graph_json,
ndjson = EXCLUDED.ndjson
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "scan_id", document.ScanId);
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "generated_at_utc", document.GeneratedAtUtc);
AddParameter(cmd, "graph_json", document.GraphJson);
AddTextArrayParameter(cmd, "ndjson", document.Ndjson?.ToArray() ?? Array.Empty<string>());
},
cancellationToken);
}
private static EntryTraceDocument MapEntryTrace(NpgsqlDataReader reader) => new()
{
ScanId = reader.GetString(reader.GetOrdinal("scan_id")),
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
GeneratedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("generated_at_utc")),
GraphJson = reader.GetString(reader.GetOrdinal("graph_json")),
Ndjson = reader.GetFieldValue<string[]>(reader.GetOrdinal("ndjson")).ToList()
};
}

View File

@@ -1,36 +1,86 @@
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class ImageRepository
public sealed class ImageRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.images";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public ImageRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null)
public ImageRepository(ScannerDataSource dataSource, ILogger<ImageRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task UpsertAsync(ImageDocument document, CancellationToken cancellationToken)
public Task UpsertAsync(ImageDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
document.LastSeenAtUtc = _timeProvider.GetUtcNow().UtcDateTime;
var updateOptions = new ReplaceOptions { IsUpsert = true };
await _collections.Images
.ReplaceOneAsync(x => x.ImageDigest == document.ImageDigest, document, updateOptions, cancellationToken)
.ConfigureAwait(false);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.LastSeenAtUtc = now;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var sql = $"""
INSERT INTO {Table} (
image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
)
VALUES (
@image_digest, @repository, @tag, @architecture, @created_at_utc, @last_seen_at_utc
)
ON CONFLICT (image_digest) DO UPDATE SET
repository = EXCLUDED.repository,
tag = EXCLUDED.tag,
architecture = EXCLUDED.architecture,
last_seen_at_utc = EXCLUDED.last_seen_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "repository", document.Repository);
AddParameter(cmd, "tag", document.Tag);
AddParameter(cmd, "architecture", document.Architecture);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
},
cancellationToken);
}
public async Task<ImageDocument?> GetAsync(string imageDigest, CancellationToken cancellationToken)
public Task<ImageDocument?> GetAsync(string imageDigest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
return await _collections.Images
.Find(x => x.ImageDigest == imageDigest)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var sql = $"""
SELECT image_digest, repository, tag, architecture, created_at_utc, last_seen_at_utc
FROM {Table}
WHERE image_digest = @image_digest
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "image_digest", imageDigest),
MapImage,
cancellationToken);
}
private static ImageDocument MapImage(NpgsqlDataReader reader) => new()
{
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
Repository = reader.GetString(reader.GetOrdinal("repository")),
Tag = GetNullableString(reader, reader.GetOrdinal("tag")),
Architecture = reader.GetString(reader.GetOrdinal("architecture")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
};
}

View File

@@ -1,64 +1,112 @@
using MongoDB.Bson;
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class JobRepository
public sealed class JobRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.jobs";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public JobRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null)
public JobRepository(ScannerDataSource dataSource, ILogger<JobRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task<JobDocument> InsertAsync(JobDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
document.CreatedAtUtc = _timeProvider.GetUtcNow().UtcDateTime;
await _collections.Jobs.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
return document;
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = now;
document.HeartbeatAtUtc = now;
var sql = $"""
INSERT INTO {Table} (
id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
)
VALUES (
@id, @kind, @state::job_state, @args::jsonb, @created_at_utc, @started_at_utc, @completed_at_utc, @heartbeat_at_utc, @error
)
RETURNING id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
""";
var inserted = await QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "kind", document.Kind);
AddParameter(cmd, "state", document.State.ToString());
AddJsonbParameter(cmd, "args", document.ArgumentsJson);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "started_at_utc", document.StartedAtUtc);
AddParameter(cmd, "completed_at_utc", document.CompletedAtUtc);
AddParameter(cmd, "heartbeat_at_utc", document.HeartbeatAtUtc);
AddParameter(cmd, "error", document.Error);
},
MapJob,
cancellationToken).ConfigureAwait(false);
return inserted ?? document;
}
public async Task<bool> TryTransitionAsync(string jobId, JobState expected, JobState next, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
var now = _timeProvider.GetUtcNow().UtcDateTime;
var update = Builders<JobDocument>.Update
.Set(x => x.State, next)
.Set(x => x.HeartbeatAtUtc, now);
if (next == JobState.Running)
{
update = update.Set(x => x.StartedAtUtc, now);
}
var sql = $"""
UPDATE {Table}
SET state = @next::job_state,
heartbeat_at_utc = @heartbeat,
started_at_utc = CASE
WHEN @next = 'Running' THEN COALESCE(started_at_utc, @heartbeat)
ELSE started_at_utc END,
completed_at_utc = CASE
WHEN @next IN ('Succeeded','Failed','Cancelled') THEN @heartbeat
ELSE completed_at_utc END
WHERE id = @id AND state = @expected::job_state
""";
if (next is JobState.Succeeded or JobState.Failed or JobState.Cancelled)
{
update = update.Set(x => x.CompletedAtUtc, now);
}
var affected = await ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", jobId);
AddParameter(cmd, "expected", expected.ToString());
AddParameter(cmd, "next", next.ToString());
AddParameter(cmd, "heartbeat", now);
},
cancellationToken).ConfigureAwait(false);
var result = await _collections.Jobs.UpdateOneAsync(
Builders<JobDocument>.Filter.And(
Builders<JobDocument>.Filter.Eq(x => x.Id, jobId),
Builders<JobDocument>.Filter.Eq(x => x.State, expected)),
update,
cancellationToken: cancellationToken).ConfigureAwait(false);
return result.ModifiedCount == 1;
return affected == 1;
}
public async Task<JobDocument?> GetAsync(string jobId, CancellationToken cancellationToken)
public Task<JobDocument?> GetAsync(string jobId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(jobId);
return await _collections.Jobs
.Find(x => x.Id == jobId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var sql = $"""
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
FROM {Table}
WHERE id = @id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "id", jobId),
MapJob,
cancellationToken);
}
public Task<List<JobDocument>> ListStaleAsync(TimeSpan heartbeatThreshold, CancellationToken cancellationToken)
@@ -69,10 +117,35 @@ public sealed class JobRepository
}
var cutoff = _timeProvider.GetUtcNow().UtcDateTime - heartbeatThreshold;
var filter = Builders<JobDocument>.Filter.And(
Builders<JobDocument>.Filter.Eq(x => x.State, JobState.Running),
Builders<JobDocument>.Filter.Lt(x => x.HeartbeatAtUtc, cutoff));
return _collections.Jobs.Find(filter).ToListAsync(cancellationToken);
var sql = $"""
SELECT id, kind, state, args, created_at_utc, started_at_utc, completed_at_utc, heartbeat_at_utc, error
FROM {Table}
WHERE state = 'Running'::job_state
AND heartbeat_at_utc < @cutoff
ORDER BY heartbeat_at_utc
""";
return QueryAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "cutoff", cutoff),
MapJob,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static JobDocument MapJob(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
Kind = reader.GetString(reader.GetOrdinal("kind")),
State = Enum.TryParse<JobState>(reader.GetString(reader.GetOrdinal("state")), true, out var state)
? state
: JobState.Pending,
ArgumentsJson = reader.GetString(reader.GetOrdinal("args")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
StartedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("started_at_utc"))?.UtcDateTime,
CompletedAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("completed_at_utc"))?.UtcDateTime,
HeartbeatAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("heartbeat_at_utc"))?.UtcDateTime,
Error = GetNullableString(reader, reader.GetOrdinal("error"))
};
}

View File

@@ -1,36 +1,79 @@
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LayerRepository
public sealed class LayerRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.layers";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public LayerRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null)
public LayerRepository(ScannerDataSource dataSource, ILogger<LayerRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task UpsertAsync(LayerDocument document, CancellationToken cancellationToken)
public Task UpsertAsync(LayerDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
document.LastSeenAtUtc = _timeProvider.GetUtcNow().UtcDateTime;
var options = new ReplaceOptions { IsUpsert = true };
await _collections.Layers
.ReplaceOneAsync(x => x.LayerDigest == document.LayerDigest, document, options, cancellationToken)
.ConfigureAwait(false);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.LastSeenAtUtc = now;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var sql = $"""
INSERT INTO {Table} (layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc)
VALUES (@layer_digest, @media_type, @size_bytes, @created_at_utc, @last_seen_at_utc)
ON CONFLICT (layer_digest) DO UPDATE SET
media_type = EXCLUDED.media_type,
size_bytes = EXCLUDED.size_bytes,
last_seen_at_utc = EXCLUDED.last_seen_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "layer_digest", document.LayerDigest);
AddParameter(cmd, "media_type", document.MediaType);
AddParameter(cmd, "size_bytes", document.SizeBytes);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
AddParameter(cmd, "last_seen_at_utc", document.LastSeenAtUtc);
},
cancellationToken);
}
public async Task<LayerDocument?> GetAsync(string layerDigest, CancellationToken cancellationToken)
public Task<LayerDocument?> GetAsync(string layerDigest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest);
return await _collections.Layers
.Find(x => x.LayerDigest == layerDigest)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var sql = $"""
SELECT layer_digest, media_type, size_bytes, created_at_utc, last_seen_at_utc
FROM {Table}
WHERE layer_digest = @layer_digest
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "layer_digest", layerDigest),
MapLayer,
cancellationToken);
}
private static LayerDocument MapLayer(NpgsqlDataReader reader) => new()
{
LayerDigest = reader.GetString(reader.GetOrdinal("layer_digest")),
MediaType = reader.GetString(reader.GetOrdinal("media_type")),
SizeBytes = reader.GetInt64(reader.GetOrdinal("size_bytes")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
LastSeenAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("last_seen_at_utc")),
};
}

View File

@@ -1,36 +1,77 @@
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LifecycleRuleRepository
public sealed class LifecycleRuleRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.lifecycle_rules";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private readonly TimeProvider _timeProvider;
public LifecycleRuleRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null)
public LifecycleRuleRepository(ScannerDataSource dataSource, ILogger<LifecycleRuleRepository> logger, TimeProvider? timeProvider = null)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
_timeProvider = timeProvider ?? TimeProvider.System;
}
public async Task UpsertAsync(LifecycleRuleDocument document, CancellationToken cancellationToken)
public Task UpsertAsync(LifecycleRuleDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var now = _timeProvider.GetUtcNow().UtcDateTime;
document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc;
var options = new ReplaceOptions { IsUpsert = true };
await _collections.LifecycleRules
.ReplaceOneAsync(x => x.Id == document.Id, document, options, cancellationToken)
.ConfigureAwait(false);
var sql = $"""
INSERT INTO {Table} (id, artifact_id, class, expires_at_utc, created_at_utc)
VALUES (@id, @artifact_id, @class, @expires_at_utc, @created_at_utc)
ON CONFLICT (id) DO UPDATE SET
artifact_id = EXCLUDED.artifact_id,
class = EXCLUDED.class,
expires_at_utc = EXCLUDED.expires_at_utc
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "artifact_id", document.ArtifactId);
AddParameter(cmd, "class", document.Class);
AddParameter(cmd, "expires_at_utc", document.ExpiresAtUtc);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
},
cancellationToken);
}
public Task<List<LifecycleRuleDocument>> ListExpiredAsync(DateTime utcNow, CancellationToken cancellationToken)
{
var filter = Builders<LifecycleRuleDocument>.Filter.Lt(x => x.ExpiresAtUtc, utcNow);
return _collections.LifecycleRules
.Find(filter)
.ToListAsync(cancellationToken);
var sql = $"""
SELECT id, artifact_id, class, expires_at_utc, created_at_utc
FROM {Table}
WHERE expires_at_utc IS NOT NULL AND expires_at_utc < @now
ORDER BY expires_at_utc
""";
return QueryAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "now", utcNow),
MapRule,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static LifecycleRuleDocument MapRule(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
Class = reader.GetString(reader.GetOrdinal("class")),
ExpiresAtUtc = GetNullableDateTimeOffset(reader, reader.GetOrdinal("expires_at_utc"))?.UtcDateTime,
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
};
}

View File

@@ -1,32 +1,80 @@
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class LinkRepository
public sealed class LinkRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.links";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
public LinkRepository(MongoCollectionProvider collections)
public LinkRepository(ScannerDataSource dataSource, ILogger<LinkRepository> logger)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
}
public async Task UpsertAsync(LinkDocument document, CancellationToken cancellationToken)
public Task UpsertAsync(LinkDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var options = new ReplaceOptions { IsUpsert = true };
await _collections.Links
.ReplaceOneAsync(x => x.Id == document.Id, document, options, cancellationToken)
.ConfigureAwait(false);
var sql = $"""
INSERT INTO {Table} (id, from_type, from_digest, artifact_id, created_at_utc)
VALUES (@id, @from_type, @from_digest, @artifact_id, @created_at_utc)
ON CONFLICT (id) DO UPDATE SET
from_type = EXCLUDED.from_type,
from_digest = EXCLUDED.from_digest,
artifact_id = EXCLUDED.artifact_id
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", document.Id);
AddParameter(cmd, "from_type", document.FromType.ToString());
AddParameter(cmd, "from_digest", document.FromDigest);
AddParameter(cmd, "artifact_id", document.ArtifactId);
AddParameter(cmd, "created_at_utc", document.CreatedAtUtc);
},
cancellationToken);
}
public Task<List<LinkDocument>> ListBySourceAsync(LinkSourceType type, string digest, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(digest);
return _collections.Links
.Find(x => x.FromType == type && x.FromDigest == digest)
.ToListAsync(cancellationToken);
var sql = $"""
SELECT id, from_type, from_digest, artifact_id, created_at_utc
FROM {Table}
WHERE from_type = @from_type AND from_digest = @from_digest
ORDER BY created_at_utc DESC, id
""";
return QueryAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "from_type", type.ToString());
AddParameter(cmd, "from_digest", digest);
},
MapLink,
cancellationToken).ContinueWith(t => t.Result.ToList(), cancellationToken);
}
private static LinkDocument MapLink(NpgsqlDataReader reader) => new()
{
Id = reader.GetString(reader.GetOrdinal("id")),
FromType = Enum.TryParse<LinkSourceType>(reader.GetString(reader.GetOrdinal("from_type")), true, out var parsed)
? parsed
: LinkSourceType.Image,
FromDigest = reader.GetString(reader.GetOrdinal("from_digest")),
ArtifactId = reader.GetString(reader.GetOrdinal("artifact_id")),
CreatedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("created_at_utc")),
};
}

View File

@@ -1,33 +1,81 @@
using MongoDB.Driver;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
public sealed class RubyPackageInventoryRepository
public sealed class RubyPackageInventoryRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.ruby_packages";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
public RubyPackageInventoryRepository(MongoCollectionProvider collections)
public RubyPackageInventoryRepository(ScannerDataSource dataSource, ILogger<RubyPackageInventoryRepository> logger)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
}
public async Task<RubyPackageInventoryDocument?> GetAsync(string scanId, CancellationToken cancellationToken)
public Task<RubyPackageInventoryDocument?> GetAsync(string scanId, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
return await _collections.RubyPackages
.Find(x => x.ScanId == scanId)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var sql = $"""
SELECT scan_id, image_digest, generated_at_utc, packages
FROM {Table}
WHERE scan_id = @scan_id
""";
return QuerySingleOrDefaultAsync(
Tenant,
sql,
cmd => AddParameter(cmd, "scan_id", scanId),
MapInventory,
cancellationToken);
}
public async Task UpsertAsync(RubyPackageInventoryDocument document, CancellationToken cancellationToken)
public Task UpsertAsync(RubyPackageInventoryDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
var options = new ReplaceOptions { IsUpsert = true };
await _collections.RubyPackages
.ReplaceOneAsync(x => x.ScanId == document.ScanId, document, options, cancellationToken)
.ConfigureAwait(false);
var sql = $"""
INSERT INTO {Table} (scan_id, image_digest, generated_at_utc, packages)
VALUES (@scan_id, @image_digest, @generated_at_utc, @packages::jsonb)
ON CONFLICT (scan_id) DO UPDATE SET
image_digest = EXCLUDED.image_digest,
generated_at_utc = EXCLUDED.generated_at_utc,
packages = EXCLUDED.packages
""";
return ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "scan_id", document.ScanId);
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "generated_at_utc", document.GeneratedAtUtc);
AddJsonbParameter(cmd, "packages", JsonSerializer.Serialize(document.Packages ?? new List<RubyPackageDocument>(), JsonOptions));
},
cancellationToken);
}
private static RubyPackageInventoryDocument MapInventory(NpgsqlDataReader reader)
{
var packagesJson = reader.GetString(reader.GetOrdinal("packages"));
var packages = string.IsNullOrWhiteSpace(packagesJson)
? new List<RubyPackageDocument>()
: JsonSerializer.Deserialize<List<RubyPackageDocument>>(packagesJson, JsonOptions) ?? new List<RubyPackageDocument>();
return new RubyPackageInventoryDocument
{
ScanId = reader.GetString(reader.GetOrdinal("scan_id")),
ImageDigest = GetNullableString(reader, reader.GetOrdinal("image_digest")),
GeneratedAtUtc = reader.GetFieldValue<DateTime>(reader.GetOrdinal("generated_at_utc")),
Packages = packages
};
}
}

View File

@@ -1,21 +1,25 @@
using System.Collections.Generic;
using System.Linq;
using MongoDB.Driver;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Infrastructure.Postgres.Repositories;
using StellaOps.Scanner.Storage.Catalog;
using StellaOps.Scanner.Storage.Mongo;
using StellaOps.Scanner.Storage.Postgres;
namespace StellaOps.Scanner.Storage.Repositories;
/// <summary>
/// Repository responsible for persisting runtime events.
/// </summary>
public sealed class RuntimeEventRepository
public sealed class RuntimeEventRepository : RepositoryBase<ScannerDataSource>
{
private readonly MongoCollectionProvider _collections;
private const string Tenant = "";
private string Table => $"{SchemaName}.runtime_events";
private string SchemaName => DataSource.SchemaName ?? ScannerDataSource.DefaultSchema;
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web);
public RuntimeEventRepository(MongoCollectionProvider collections)
public RuntimeEventRepository(ScannerDataSource dataSource, ILogger<RuntimeEventRepository> logger)
: base(dataSource, logger)
{
_collections = collections ?? throw new ArgumentNullException(nameof(collections));
}
public async Task<RuntimeEventInsertResult> InsertAsync(
@@ -28,27 +32,70 @@ public sealed class RuntimeEventRepository
return RuntimeEventInsertResult.Empty;
}
try
var sql = $"""
INSERT INTO {Table} (
id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
platform, namespace, pod, container, container_id, image_ref, image_digest,
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
)
VALUES (
@id, @event_id, @schema_version, @tenant, @node, @kind, @when, @received_at, @expires_at,
@platform, @namespace, @pod, @container, @container_id, @image_ref, @image_digest,
@engine, @engine_version, @baseline_digest, @image_signed, @sbom_referrer, @build_id, @payload::jsonb
)
ON CONFLICT (event_id) DO NOTHING
""";
var inserted = 0;
var duplicates = 0;
foreach (var document in documents)
{
await _collections.RuntimeEvents.InsertManyAsync(
documents,
new InsertManyOptions { IsOrdered = false },
cancellationToken.ThrowIfCancellationRequested();
var id = string.IsNullOrWhiteSpace(document.Id) ? Guid.NewGuid().ToString("N") : document.Id;
var rows = await ExecuteAsync(
Tenant,
sql,
cmd =>
{
AddParameter(cmd, "id", id);
AddParameter(cmd, "event_id", document.EventId);
AddParameter(cmd, "schema_version", document.SchemaVersion);
AddParameter(cmd, "tenant", document.Tenant);
AddParameter(cmd, "node", document.Node);
AddParameter(cmd, "kind", document.Kind);
AddParameter(cmd, "when", document.When);
AddParameter(cmd, "received_at", document.ReceivedAt);
AddParameter(cmd, "expires_at", document.ExpiresAt);
AddParameter(cmd, "platform", document.Platform);
AddParameter(cmd, "namespace", document.Namespace);
AddParameter(cmd, "pod", document.Pod);
AddParameter(cmd, "container", document.Container);
AddParameter(cmd, "container_id", document.ContainerId);
AddParameter(cmd, "image_ref", document.ImageRef);
AddParameter(cmd, "image_digest", document.ImageDigest);
AddParameter(cmd, "engine", document.Engine);
AddParameter(cmd, "engine_version", document.EngineVersion);
AddParameter(cmd, "baseline_digest", document.BaselineDigest);
AddParameter(cmd, "image_signed", document.ImageSigned);
AddParameter(cmd, "sbom_referrer", document.SbomReferrer);
AddParameter(cmd, "build_id", document.BuildId);
AddJsonbParameter(cmd, "payload", document.PayloadJson);
},
cancellationToken).ConfigureAwait(false);
return new RuntimeEventInsertResult(documents.Count, 0);
}
catch (MongoBulkWriteException<RuntimeEventDocument> ex)
{
var duplicates = ex.WriteErrors
.Count(error => error.Category == ServerErrorCategory.DuplicateKey);
var inserted = documents.Count - duplicates;
if (inserted < 0)
if (rows > 0)
{
inserted = 0;
inserted++;
}
else
{
duplicates++;
}
return new RuntimeEventInsertResult(inserted, duplicates);
}
return new RuntimeEventInsertResult(inserted, duplicates);
}
public async Task<IReadOnlyDictionary<string, RuntimeBuildIdObservation>> GetRecentBuildIdsAsync(
@@ -73,35 +120,36 @@ public sealed class RuntimeEventRepository
return new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
}
var results = new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
var limit = Math.Max(1, maxPerImage);
var sql = $"""
SELECT image_digest, build_id, "when"
FROM {Table}
WHERE image_digest = ANY(@digests)
AND build_id IS NOT NULL
AND build_id <> ''
ORDER BY image_digest, "when" DESC
""";
foreach (var digest in normalized)
{
var filter = Builders<RuntimeEventDocument>.Filter.And(
Builders<RuntimeEventDocument>.Filter.Eq(doc => doc.ImageDigest, digest),
Builders<RuntimeEventDocument>.Filter.Ne(doc => doc.BuildId, null),
Builders<RuntimeEventDocument>.Filter.Ne(doc => doc.BuildId, string.Empty));
var documents = await _collections.RuntimeEvents
.Find(filter)
.SortByDescending(doc => doc.When)
.Limit(limit * 4)
.Project(doc => new { doc.BuildId, doc.When })
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
if (documents.Count == 0)
var rows = await QueryAsync(
Tenant,
sql,
cmd => AddTextArrayParameter(cmd, "digests", normalized),
reader => new
{
continue;
}
ImageDigest = reader.GetString(reader.GetOrdinal("image_digest")),
BuildId = reader.GetString(reader.GetOrdinal("build_id")),
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when"))
},
cancellationToken).ConfigureAwait(false);
var buildIds = documents
.Select(doc => doc.BuildId)
var result = new Dictionary<string, RuntimeBuildIdObservation>(StringComparer.Ordinal);
foreach (var group in rows.GroupBy(r => r.ImageDigest, StringComparer.OrdinalIgnoreCase))
{
var buildIds = group
.Select(r => r.BuildId)
.Where(id => !string.IsNullOrWhiteSpace(id))
.Distinct(StringComparer.OrdinalIgnoreCase)
.Take(limit)
.Select(id => id!.Trim().ToLowerInvariant())
.Take(maxPerImage)
.Select(id => id.Trim().ToLowerInvariant())
.ToArray();
if (buildIds.Length == 0)
@@ -109,15 +157,82 @@ public sealed class RuntimeEventRepository
continue;
}
var observedAt = documents
.Where(doc => !string.IsNullOrWhiteSpace(doc.BuildId))
.Select(doc => doc.When)
.FirstOrDefault();
results[digest] = new RuntimeBuildIdObservation(digest, buildIds, observedAt);
var observedAt = group.Select(r => r.When).FirstOrDefault();
result[group.Key.ToLowerInvariant()] = new RuntimeBuildIdObservation(group.Key, buildIds, observedAt);
}
return results;
return result;
}
public Task<int> CountAsync(CancellationToken cancellationToken)
{
var sql = $"""
SELECT COUNT(*) FROM {Table}
""";
return ExecuteScalarAsync<int>(
Tenant,
sql,
null,
cancellationToken);
}
public Task TruncateAsync(CancellationToken cancellationToken)
{
var sql = $"""
TRUNCATE TABLE {Table} RESTART IDENTITY CASCADE
""";
return ExecuteAsync(Tenant, sql, null, cancellationToken);
}
public Task<IReadOnlyList<RuntimeEventDocument>> ListAsync(CancellationToken cancellationToken)
{
var sql = $"""
SELECT id, event_id, schema_version, tenant, node, kind, "when", received_at, expires_at,
platform, namespace, pod, container, container_id, image_ref, image_digest,
engine, engine_version, baseline_digest, image_signed, sbom_referrer, build_id, payload
FROM {Table}
ORDER BY received_at
""";
return QueryAsync(
Tenant,
sql,
null,
MapRuntimeEvent,
cancellationToken);
}
private static RuntimeEventDocument MapRuntimeEvent(NpgsqlDataReader reader)
{
var payloadOrdinal = reader.GetOrdinal("payload");
return new RuntimeEventDocument
{
Id = reader.GetString(reader.GetOrdinal("id")),
EventId = reader.GetString(reader.GetOrdinal("event_id")),
SchemaVersion = reader.GetString(reader.GetOrdinal("schema_version")),
Tenant = reader.GetString(reader.GetOrdinal("tenant")),
Node = reader.GetString(reader.GetOrdinal("node")),
Kind = reader.GetString(reader.GetOrdinal("kind")),
When = reader.GetFieldValue<DateTime>(reader.GetOrdinal("when")),
ReceivedAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("received_at")),
ExpiresAt = reader.GetFieldValue<DateTime>(reader.GetOrdinal("expires_at")),
Platform = GetNullableString(reader, reader.GetOrdinal("platform")),
Namespace = GetNullableString(reader, reader.GetOrdinal("namespace")),
Pod = GetNullableString(reader, reader.GetOrdinal("pod")),
Container = GetNullableString(reader, reader.GetOrdinal("container")),
ContainerId = GetNullableString(reader, reader.GetOrdinal("container_id")),
ImageRef = GetNullableString(reader, reader.GetOrdinal("image_ref")),
ImageDigest = GetNullableString(reader, reader.GetOrdinal("image_digest")),
Engine = GetNullableString(reader, reader.GetOrdinal("engine")),
EngineVersion = GetNullableString(reader, reader.GetOrdinal("engine_version")),
BaselineDigest = GetNullableString(reader, reader.GetOrdinal("baseline_digest")),
ImageSigned = GetNullableBoolean(reader, reader.GetOrdinal("image_signed")),
SbomReferrer = GetNullableString(reader, reader.GetOrdinal("sbom_referrer")),
BuildId = GetNullableString(reader, reader.GetOrdinal("build_id")),
PayloadJson = reader.IsDBNull(payloadOrdinal) ? "{}" : reader.GetString(payloadOrdinal)
};
}
}

View File

@@ -3,6 +3,7 @@ namespace StellaOps.Scanner.Storage;
public static class ScannerStorageDefaults
{
public const string DefaultDatabaseName = "scanner";
public const string DefaultSchemaName = "scanner";
public const string DefaultBucketName = "stellaops";
public const string DefaultRootPrefix = "scanner";

View File

@@ -1,76 +1,39 @@
using System;
using System.Collections.Generic;
using MongoDB.Driver;
namespace StellaOps.Scanner.Storage;
public sealed class ScannerStorageOptions
{
public MongoOptions Mongo { get; set; } = new();
public ObjectStoreOptions ObjectStore { get; set; } = new();
public DualWriteOptions DualWrite { get; set; } = new();
public void EnsureValid()
{
Mongo.EnsureValid();
ObjectStore.EnsureValid();
DualWrite.EnsureValid();
}
}
public sealed class MongoOptions
{
public string ConnectionString { get; set; } = string.Empty;
public string? DatabaseName { get; set; }
= null;
public TimeSpan CommandTimeout { get; set; }
= TimeSpan.FromSeconds(30);
public bool UseMajorityReadConcern { get; set; }
= true;
public bool UseMajorityWriteConcern { get; set; }
= true;
public string ResolveDatabaseName()
{
if (!string.IsNullOrWhiteSpace(DatabaseName))
{
return DatabaseName.Trim();
}
if (!string.IsNullOrWhiteSpace(ConnectionString))
{
var url = MongoUrl.Create(ConnectionString);
if (!string.IsNullOrWhiteSpace(url.DatabaseName))
{
return url.DatabaseName;
}
}
return ScannerStorageDefaults.DefaultDatabaseName;
}
public void EnsureValid()
{
if (string.IsNullOrWhiteSpace(ConnectionString))
{
throw new InvalidOperationException("Scanner storage Mongo connection string is not configured.");
}
if (CommandTimeout <= TimeSpan.Zero)
{
throw new InvalidOperationException("Scanner storage Mongo command timeout must be positive.");
}
_ = ResolveDatabaseName();
}
}
using StellaOps.Infrastructure.Postgres.Options;
namespace StellaOps.Scanner.Storage;
public sealed class ScannerStorageOptions
{
public PostgresOptions Postgres { get; set; } = new() { ConnectionString = string.Empty, SchemaName = ScannerStorageDefaults.DefaultSchemaName };
public ObjectStoreOptions ObjectStore { get; set; } = new();
public DualWriteOptions DualWrite { get; set; } = new();
public void EnsureValid()
{
if (Postgres is null)
{
throw new InvalidOperationException("Scanner storage Postgres options must be configured.");
}
if (string.IsNullOrWhiteSpace(Postgres.ConnectionString))
{
throw new InvalidOperationException("Scanner storage Postgres connection string is not configured.");
}
if (string.IsNullOrWhiteSpace(Postgres.SchemaName))
{
Postgres.SchemaName = ScannerStorageDefaults.DefaultSchemaName;
}
ObjectStore.EnsureValid();
DualWrite.EnsureValid();
}
}
public sealed class ObjectStoreOptions
{
private static readonly HashSet<string> S3Drivers = new(StringComparer.OrdinalIgnoreCase)
@@ -194,20 +157,20 @@ public sealed class RustFsOptions
}
}
}
public sealed class DualWriteOptions
{
public bool Enabled { get; set; }
= false;
public string? MirrorBucket { get; set; }
= null;
public void EnsureValid()
{
if (Enabled && string.IsNullOrWhiteSpace(MirrorBucket))
{
throw new InvalidOperationException("Dual-write mirror bucket must be configured when enabled.");
}
}
}
public sealed class DualWriteOptions
{
public bool Enabled { get; set; }
= false;
public string? MirrorBucket { get; set; }
= null;
public void EnsureValid()
{
if (Enabled && string.IsNullOrWhiteSpace(MirrorBucket))
{
throw new InvalidOperationException("Dual-write mirror bucket must be configured when enabled.");
}
}
}

View File

@@ -7,16 +7,20 @@
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="AWSSDK.S3" Version="3.7.305.6" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0" />
<PackageReference Include="Npgsql" Version="9.0.2" />
</ItemGroup>
<ItemGroup>
<EmbeddedResource Include="Postgres\Migrations\**\*.sql" LogicalName="%(RecursiveDir)%(Filename)%(Extension)" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\\StellaOps.Scanner.EntryTrace\\StellaOps.Scanner.EntryTrace.csproj" />
<ProjectReference Include="..\\StellaOps.Scanner.Core\\StellaOps.Scanner.Core.csproj" />
<ProjectReference Include="..\\..\\..\\__Libraries\\StellaOps.Infrastructure.Postgres\\StellaOps.Infrastructure.Postgres.csproj" />
</ItemGroup>
</Project>