up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
This commit is contained in:
179
src/Signals/StellaOps.Signals/Models/EdgeBundleDocument.cs
Normal file
179
src/Signals/StellaOps.Signals/Models/EdgeBundleDocument.cs
Normal file
@@ -0,0 +1,179 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Edge bundle document for storing ingested edge bundles.
|
||||
/// </summary>
|
||||
public sealed class EdgeBundleDocument
|
||||
{
|
||||
public string Id { get; set; } = Guid.NewGuid().ToString("N");
|
||||
|
||||
/// <summary>
|
||||
/// Bundle identifier from the DSSE envelope.
|
||||
/// </summary>
|
||||
public string BundleId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Graph hash this bundle is associated with.
|
||||
/// </summary>
|
||||
public string GraphHash { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Tenant identifier for isolation.
|
||||
/// </summary>
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Reason for this bundle (RuntimeHits, InitArray, ThirdParty, Contested, Revoked, etc.).
|
||||
/// </summary>
|
||||
public string BundleReason { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Custom reason description if BundleReason is Custom.
|
||||
/// </summary>
|
||||
public string? CustomReason { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Edges in this bundle.
|
||||
/// </summary>
|
||||
public List<EdgeBundleEdgeDocument> Edges { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Content hash of the bundle (sha256:...).
|
||||
/// </summary>
|
||||
public string ContentHash { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// DSSE envelope digest.
|
||||
/// </summary>
|
||||
public string? DsseDigest { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// CAS URI for the bundle JSON.
|
||||
/// </summary>
|
||||
public string? CasUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// CAS URI for the DSSE envelope.
|
||||
/// </summary>
|
||||
public string? DsseCasUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this bundle has been verified.
|
||||
/// </summary>
|
||||
public bool Verified { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Rekor log index if published.
|
||||
/// </summary>
|
||||
public long? RekorLogIndex { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Count of revoked edges in this bundle.
|
||||
/// </summary>
|
||||
public int RevokedCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the bundle was ingested.
|
||||
/// </summary>
|
||||
public DateTimeOffset IngestedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When the bundle was generated.
|
||||
/// </summary>
|
||||
public DateTimeOffset GeneratedAt { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Individual edge within an edge bundle document.
|
||||
/// </summary>
|
||||
public sealed class EdgeBundleEdgeDocument
|
||||
{
|
||||
/// <summary>
|
||||
/// Source function/method ID.
|
||||
/// </summary>
|
||||
public string From { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Target function/method ID.
|
||||
/// </summary>
|
||||
public string To { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Edge kind (call, callvirt, invokestatic, etc.).
|
||||
/// </summary>
|
||||
public string Kind { get; set; } = "call";
|
||||
|
||||
/// <summary>
|
||||
/// Reason for inclusion in this bundle.
|
||||
/// </summary>
|
||||
public string Reason { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Whether this edge is revoked (patched/removed).
|
||||
/// </summary>
|
||||
public bool Revoked { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Confidence level (0.0-1.0).
|
||||
/// </summary>
|
||||
public double Confidence { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Package URL of the target.
|
||||
/// </summary>
|
||||
public string? Purl { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Symbol digest of the target.
|
||||
/// </summary>
|
||||
public string? SymbolDigest { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Evidence URI for this edge.
|
||||
/// </summary>
|
||||
public string? Evidence { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reference to an edge bundle attached to a reachability fact.
|
||||
/// </summary>
|
||||
public sealed class EdgeBundleReference
|
||||
{
|
||||
/// <summary>
|
||||
/// Bundle identifier.
|
||||
/// </summary>
|
||||
public string BundleId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle reason.
|
||||
/// </summary>
|
||||
public string BundleReason { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// CAS URI for the bundle.
|
||||
/// </summary>
|
||||
public string? CasUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// DSSE CAS URI.
|
||||
/// </summary>
|
||||
public string? DsseCasUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of edges in the bundle.
|
||||
/// </summary>
|
||||
public int EdgeCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of revoked edges.
|
||||
/// </summary>
|
||||
public int RevokedCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the bundle has been verified.
|
||||
/// </summary>
|
||||
public bool Verified { get; set; }
|
||||
}
|
||||
232
src/Signals/StellaOps.Signals/Models/ProcSnapshotDocument.cs
Normal file
232
src/Signals/StellaOps.Signals/Models/ProcSnapshotDocument.cs
Normal file
@@ -0,0 +1,232 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Document representing a proc snapshot for Java/.NET/PHP runtime parity.
|
||||
/// Captures runtime-observed classpath, loaded assemblies, and autoload paths.
|
||||
/// </summary>
|
||||
public sealed class ProcSnapshotDocument
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique identifier for this snapshot (format: {tenant}/{image_digest}/{snapshot_hash}).
|
||||
/// </summary>
|
||||
public required string Id { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Tenant identifier for multi-tenancy isolation.
|
||||
/// </summary>
|
||||
public required string Tenant { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Image digest of the container this snapshot was captured from.
|
||||
/// </summary>
|
||||
[JsonPropertyName("imageDigest")]
|
||||
public required string ImageDigest { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Node identifier where the snapshot was captured.
|
||||
/// </summary>
|
||||
public string? Node { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Container ID where the process was running.
|
||||
/// </summary>
|
||||
public string? ContainerId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Process ID at capture time.
|
||||
/// </summary>
|
||||
public int Pid { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Process entrypoint command.
|
||||
/// </summary>
|
||||
public string? Entrypoint { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Runtime type: java, dotnet, php.
|
||||
/// </summary>
|
||||
public required string RuntimeType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Runtime version (e.g., "17.0.2", "8.0.0", "8.2.0").
|
||||
/// </summary>
|
||||
public string? RuntimeVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Java classpath entries (jar paths, directories).
|
||||
/// </summary>
|
||||
[JsonPropertyName("classpath")]
|
||||
public IReadOnlyList<ClasspathEntry> Classpath { get; init; } = Array.Empty<ClasspathEntry>();
|
||||
|
||||
/// <summary>
|
||||
/// .NET loaded assemblies with RID-graph resolution.
|
||||
/// </summary>
|
||||
[JsonPropertyName("loadedAssemblies")]
|
||||
public IReadOnlyList<LoadedAssemblyEntry> LoadedAssemblies { get; init; } = Array.Empty<LoadedAssemblyEntry>();
|
||||
|
||||
/// <summary>
|
||||
/// PHP autoload paths from composer autoloader.
|
||||
/// </summary>
|
||||
[JsonPropertyName("autoloadPaths")]
|
||||
public IReadOnlyList<AutoloadPathEntry> AutoloadPaths { get; init; } = Array.Empty<AutoloadPathEntry>();
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp when the snapshot was captured.
|
||||
/// </summary>
|
||||
public DateTimeOffset CapturedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp when the snapshot was stored.
|
||||
/// </summary>
|
||||
public DateTimeOffset StoredAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Expiration timestamp for TTL-based cleanup.
|
||||
/// </summary>
|
||||
public DateTimeOffset? ExpiresAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional annotations/metadata.
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, string>? Annotations { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Java classpath entry representing a JAR or directory.
|
||||
/// </summary>
|
||||
public sealed class ClasspathEntry
|
||||
{
|
||||
/// <summary>
|
||||
/// Path to the JAR file or classpath directory.
|
||||
/// </summary>
|
||||
public required string Path { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Type: jar, directory, jmod.
|
||||
/// </summary>
|
||||
public string? Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the JAR file (null for directories).
|
||||
/// </summary>
|
||||
public string? Sha256 { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Maven coordinate if resolvable (e.g., "org.springframework:spring-core:5.3.20").
|
||||
/// </summary>
|
||||
public string? MavenCoordinate { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Package URL (PURL) if resolvable.
|
||||
/// </summary>
|
||||
public string? Purl { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Size in bytes.
|
||||
/// </summary>
|
||||
public long? SizeBytes { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// .NET loaded assembly entry with RID-graph context.
|
||||
/// </summary>
|
||||
public sealed class LoadedAssemblyEntry
|
||||
{
|
||||
/// <summary>
|
||||
/// Assembly name (e.g., "System.Text.Json").
|
||||
/// </summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Assembly version (e.g., "8.0.0.0").
|
||||
/// </summary>
|
||||
public string? Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Full path to the loaded DLL.
|
||||
/// </summary>
|
||||
public required string Path { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the assembly file.
|
||||
/// </summary>
|
||||
public string? Sha256 { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// NuGet package ID if resolvable.
|
||||
/// </summary>
|
||||
public string? NuGetPackage { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// NuGet package version if resolvable.
|
||||
/// </summary>
|
||||
public string? NuGetVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Package URL (PURL) if resolvable.
|
||||
/// </summary>
|
||||
public string? Purl { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Runtime identifier (RID) for platform-specific assemblies.
|
||||
/// </summary>
|
||||
public string? Rid { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this assembly was loaded from the shared framework.
|
||||
/// </summary>
|
||||
public bool? IsFrameworkAssembly { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Source from deps.json resolution: compile, runtime, native.
|
||||
/// </summary>
|
||||
public string? DepsSource { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// PHP autoload path entry from Composer.
|
||||
/// </summary>
|
||||
public sealed class AutoloadPathEntry
|
||||
{
|
||||
/// <summary>
|
||||
/// Namespace prefix (PSR-4) or class name (classmap).
|
||||
/// </summary>
|
||||
public string? Namespace { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Autoload type: psr-4, psr-0, classmap, files.
|
||||
/// </summary>
|
||||
public required string Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to the autoloaded file or directory.
|
||||
/// </summary>
|
||||
public required string Path { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Composer package name if resolvable.
|
||||
/// </summary>
|
||||
public string? ComposerPackage { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Composer package version if resolvable.
|
||||
/// </summary>
|
||||
public string? ComposerVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Package URL (PURL) if resolvable.
|
||||
/// </summary>
|
||||
public string? Purl { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Known runtime types for proc snapshots.
|
||||
/// </summary>
|
||||
public static class ProcSnapshotRuntimeTypes
|
||||
{
|
||||
public const string Java = "java";
|
||||
public const string DotNet = "dotnet";
|
||||
public const string Php = "php";
|
||||
}
|
||||
@@ -17,12 +17,32 @@ public sealed class ReachabilityFactDocument
|
||||
|
||||
public List<RuntimeFactDocument>? RuntimeFacts { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// CAS URI for the runtime-facts batch artifact (cas://reachability/runtime-facts/{hash}).
|
||||
/// </summary>
|
||||
public string? RuntimeFactsBatchUri { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// BLAKE3 hash of the runtime-facts batch artifact.
|
||||
/// </summary>
|
||||
public string? RuntimeFactsBatchHash { get; set; }
|
||||
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
|
||||
public ContextFacts? ContextFacts { get; set; }
|
||||
|
||||
public UncertaintyDocument? Uncertainty { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Edge bundles attached to this graph.
|
||||
/// </summary>
|
||||
public List<EdgeBundleReference>? EdgeBundles { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether any edges are quarantined (revoked) for this fact.
|
||||
/// </summary>
|
||||
public bool HasQuarantinedEdges { get; set; }
|
||||
|
||||
public double Score { get; set; }
|
||||
|
||||
public double RiskScore { get; set; }
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Persistence;
|
||||
|
||||
/// <summary>
|
||||
/// Repository for persisting and querying proc snapshot documents.
|
||||
/// </summary>
|
||||
public interface IProcSnapshotRepository
|
||||
{
|
||||
/// <summary>
|
||||
/// Upsert a proc snapshot document.
|
||||
/// </summary>
|
||||
Task<ProcSnapshotDocument> UpsertAsync(ProcSnapshotDocument document, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Get a proc snapshot by ID.
|
||||
/// </summary>
|
||||
Task<ProcSnapshotDocument?> GetByIdAsync(string id, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Get all proc snapshots for a specific image digest.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<ProcSnapshotDocument>> GetByImageDigestAsync(
|
||||
string tenant,
|
||||
string imageDigest,
|
||||
int limit = 100,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Get the most recent proc snapshot for an image digest and runtime type.
|
||||
/// </summary>
|
||||
Task<ProcSnapshotDocument?> GetLatestAsync(
|
||||
string tenant,
|
||||
string imageDigest,
|
||||
string runtimeType,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Delete expired proc snapshots.
|
||||
/// </summary>
|
||||
Task<int> DeleteExpiredAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
using System.Collections.Concurrent;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Persistence;
|
||||
|
||||
/// <summary>
|
||||
/// In-memory implementation of <see cref="IProcSnapshotRepository"/> for testing and development.
|
||||
/// </summary>
|
||||
public sealed class InMemoryProcSnapshotRepository : IProcSnapshotRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ProcSnapshotDocument> _documents = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public InMemoryProcSnapshotRepository(TimeProvider timeProvider)
|
||||
{
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public Task<ProcSnapshotDocument> UpsertAsync(ProcSnapshotDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
_documents[document.Id] = document;
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
|
||||
public Task<ProcSnapshotDocument?> GetByIdAsync(string id, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(id);
|
||||
|
||||
_documents.TryGetValue(id, out var document);
|
||||
return Task.FromResult(document);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<ProcSnapshotDocument>> GetByImageDigestAsync(
|
||||
string tenant,
|
||||
string imageDigest,
|
||||
int limit = 100,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
|
||||
|
||||
var normalizedDigest = imageDigest.ToLowerInvariant();
|
||||
var results = _documents.Values
|
||||
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(d.ImageDigest, normalizedDigest, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderByDescending(d => d.CapturedAt)
|
||||
.Take(limit)
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<ProcSnapshotDocument>>(results);
|
||||
}
|
||||
|
||||
public Task<ProcSnapshotDocument?> GetLatestAsync(
|
||||
string tenant,
|
||||
string imageDigest,
|
||||
string runtimeType,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(runtimeType);
|
||||
|
||||
var normalizedDigest = imageDigest.ToLowerInvariant();
|
||||
var result = _documents.Values
|
||||
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(d.ImageDigest, normalizedDigest, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(d.RuntimeType, runtimeType, StringComparison.OrdinalIgnoreCase))
|
||||
.OrderByDescending(d => d.CapturedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public Task<int> DeleteExpiredAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var expiredIds = _documents
|
||||
.Where(kv => kv.Value.ExpiresAt.HasValue && kv.Value.ExpiresAt.Value < now)
|
||||
.Select(kv => kv.Key)
|
||||
.ToList();
|
||||
|
||||
var deletedCount = 0;
|
||||
foreach (var id in expiredIds)
|
||||
{
|
||||
if (_documents.TryRemove(id, out _))
|
||||
{
|
||||
deletedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult(deletedCount);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Options;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Ingests edge-bundle DSSE envelopes, attaches to graph_hash, enforces quarantine for revoked edges.
|
||||
/// </summary>
|
||||
public sealed class EdgeBundleIngestionService : IEdgeBundleIngestionService
|
||||
{
|
||||
private readonly ILogger<EdgeBundleIngestionService> _logger;
|
||||
private readonly SignalsOptions _options;
|
||||
|
||||
// In-memory storage (in production, would use repository)
|
||||
private readonly ConcurrentDictionary<string, List<EdgeBundleDocument>> _bundlesByGraphHash = new();
|
||||
private readonly ConcurrentDictionary<string, HashSet<string>> _revokedEdgeKeys = new();
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
public EdgeBundleIngestionService(
|
||||
ILogger<EdgeBundleIngestionService> logger,
|
||||
IOptions<SignalsOptions> options)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
public async Task<EdgeBundleIngestResponse> IngestAsync(
|
||||
string tenantId,
|
||||
Stream bundleStream,
|
||||
Stream? dsseStream,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(bundleStream);
|
||||
|
||||
// Parse the bundle JSON
|
||||
using var bundleMs = new MemoryStream();
|
||||
await bundleStream.CopyToAsync(bundleMs, cancellationToken).ConfigureAwait(false);
|
||||
bundleMs.Position = 0;
|
||||
|
||||
var bundleJson = await JsonDocument.ParseAsync(bundleMs, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
var root = bundleJson.RootElement;
|
||||
|
||||
// Extract bundle fields
|
||||
var bundleId = GetStringOrDefault(root, "bundleId", $"bundle:{Guid.NewGuid():N}");
|
||||
var graphHash = GetStringOrDefault(root, "graphHash", string.Empty);
|
||||
var bundleReason = GetStringOrDefault(root, "bundleReason", "Custom");
|
||||
var customReason = GetStringOrDefault(root, "customReason", null);
|
||||
var generatedAtStr = GetStringOrDefault(root, "generatedAt", null);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(graphHash))
|
||||
{
|
||||
throw new InvalidOperationException("Edge bundle missing required 'graphHash' field");
|
||||
}
|
||||
|
||||
var generatedAt = !string.IsNullOrWhiteSpace(generatedAtStr)
|
||||
? DateTimeOffset.Parse(generatedAtStr)
|
||||
: DateTimeOffset.UtcNow;
|
||||
|
||||
// Parse edges
|
||||
var edges = new List<EdgeBundleEdgeDocument>();
|
||||
var revokedCount = 0;
|
||||
|
||||
if (root.TryGetProperty("edges", out var edgesElement) && edgesElement.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var edgeEl in edgesElement.EnumerateArray())
|
||||
{
|
||||
var edge = new EdgeBundleEdgeDocument
|
||||
{
|
||||
From = GetStringOrDefault(edgeEl, "from", string.Empty),
|
||||
To = GetStringOrDefault(edgeEl, "to", string.Empty),
|
||||
Kind = GetStringOrDefault(edgeEl, "kind", "call"),
|
||||
Reason = GetStringOrDefault(edgeEl, "reason", "Unknown"),
|
||||
Revoked = edgeEl.TryGetProperty("revoked", out var r) && r.GetBoolean(),
|
||||
Confidence = edgeEl.TryGetProperty("confidence", out var c) ? c.GetDouble() : 0.5,
|
||||
Purl = GetStringOrDefault(edgeEl, "purl", null),
|
||||
SymbolDigest = GetStringOrDefault(edgeEl, "symbolDigest", null),
|
||||
Evidence = GetStringOrDefault(edgeEl, "evidence", null)
|
||||
};
|
||||
|
||||
edges.Add(edge);
|
||||
|
||||
if (edge.Revoked)
|
||||
{
|
||||
revokedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute content hash
|
||||
bundleMs.Position = 0;
|
||||
var contentHash = ComputeSha256(bundleMs);
|
||||
|
||||
// Parse DSSE if provided
|
||||
string? dsseDigest = null;
|
||||
if (dsseStream is not null)
|
||||
{
|
||||
using var dsseMs = new MemoryStream();
|
||||
await dsseStream.CopyToAsync(dsseMs, cancellationToken).ConfigureAwait(false);
|
||||
dsseMs.Position = 0;
|
||||
dsseDigest = $"sha256:{ComputeSha256(dsseMs)}";
|
||||
}
|
||||
|
||||
// Build CAS URIs
|
||||
var graphHashDigest = ExtractHashDigest(graphHash);
|
||||
var casUri = $"cas://reachability/edges/{graphHashDigest}/{bundleId}";
|
||||
var dsseCasUri = dsseStream is not null ? $"{casUri}.dsse" : null;
|
||||
|
||||
// Create document
|
||||
var document = new EdgeBundleDocument
|
||||
{
|
||||
BundleId = bundleId,
|
||||
GraphHash = graphHash,
|
||||
TenantId = tenantId,
|
||||
BundleReason = bundleReason,
|
||||
CustomReason = customReason,
|
||||
Edges = edges,
|
||||
ContentHash = $"sha256:{contentHash}",
|
||||
DsseDigest = dsseDigest,
|
||||
CasUri = casUri,
|
||||
DsseCasUri = dsseCasUri,
|
||||
Verified = dsseStream is not null, // Simple verification - in production would verify signature
|
||||
RevokedCount = revokedCount,
|
||||
IngestedAt = DateTimeOffset.UtcNow,
|
||||
GeneratedAt = generatedAt
|
||||
};
|
||||
|
||||
// Store document
|
||||
var storageKey = $"{tenantId}:{graphHash}";
|
||||
_bundlesByGraphHash.AddOrUpdate(
|
||||
storageKey,
|
||||
_ => new List<EdgeBundleDocument> { document },
|
||||
(_, list) =>
|
||||
{
|
||||
// Remove existing bundle with same ID
|
||||
list.RemoveAll(b => b.BundleId == bundleId);
|
||||
list.Add(document);
|
||||
return list;
|
||||
});
|
||||
|
||||
// Update revoked edge index for quarantine enforcement
|
||||
if (revokedCount > 0)
|
||||
{
|
||||
var revokedEdges = edges.Where(e => e.Revoked).Select(e => $"{e.From}>{e.To}").ToHashSet();
|
||||
_revokedEdgeKeys.AddOrUpdate(
|
||||
storageKey,
|
||||
_ => revokedEdges,
|
||||
(_, existing) =>
|
||||
{
|
||||
foreach (var key in revokedEdges)
|
||||
{
|
||||
existing.Add(key);
|
||||
}
|
||||
return existing;
|
||||
});
|
||||
}
|
||||
|
||||
var quarantined = revokedCount > 0;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Ingested edge bundle {BundleId} for graph {GraphHash} with {EdgeCount} edges ({RevokedCount} revoked, quarantine={Quarantined})",
|
||||
bundleId, graphHash, edges.Count, revokedCount, quarantined);
|
||||
|
||||
return new EdgeBundleIngestResponse(
|
||||
bundleId,
|
||||
graphHash,
|
||||
bundleReason,
|
||||
casUri,
|
||||
dsseCasUri,
|
||||
edges.Count,
|
||||
revokedCount,
|
||||
quarantined);
|
||||
}
|
||||
|
||||
public Task<EdgeBundleDocument[]> GetBundlesForGraphAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = $"{tenantId}:{graphHash}";
|
||||
if (_bundlesByGraphHash.TryGetValue(key, out var bundles))
|
||||
{
|
||||
return Task.FromResult(bundles.ToArray());
|
||||
}
|
||||
|
||||
return Task.FromResult(Array.Empty<EdgeBundleDocument>());
|
||||
}
|
||||
|
||||
public Task<EdgeBundleEdgeDocument[]> GetRevokedEdgesAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = $"{tenantId}:{graphHash}";
|
||||
if (_bundlesByGraphHash.TryGetValue(key, out var bundles))
|
||||
{
|
||||
var revoked = bundles
|
||||
.SelectMany(b => b.Edges)
|
||||
.Where(e => e.Revoked)
|
||||
.ToArray();
|
||||
return Task.FromResult(revoked);
|
||||
}
|
||||
|
||||
return Task.FromResult(Array.Empty<EdgeBundleEdgeDocument>());
|
||||
}
|
||||
|
||||
public Task<bool> IsEdgeRevokedAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
string fromId,
|
||||
string toId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var key = $"{tenantId}:{graphHash}";
|
||||
if (_revokedEdgeKeys.TryGetValue(key, out var revokedKeys))
|
||||
{
|
||||
var edgeKey = $"{fromId}>{toId}";
|
||||
return Task.FromResult(revokedKeys.Contains(edgeKey));
|
||||
}
|
||||
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
private static string GetStringOrDefault(JsonElement element, string propertyName, string? defaultValue)
|
||||
{
|
||||
if (element.TryGetProperty(propertyName, out var prop) && prop.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return prop.GetString() ?? defaultValue ?? string.Empty;
|
||||
}
|
||||
return defaultValue ?? string.Empty;
|
||||
}
|
||||
|
||||
private static string ComputeSha256(Stream stream)
|
||||
{
|
||||
using var sha = SHA256.Create();
|
||||
var hash = sha.ComputeHash(stream);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string ExtractHashDigest(string prefixedHash)
|
||||
{
|
||||
var colonIndex = prefixedHash.IndexOf(':');
|
||||
return colonIndex >= 0 ? prefixedHash[(colonIndex + 1)..] : prefixedHash;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Response from edge bundle ingestion.
|
||||
/// </summary>
|
||||
public sealed record EdgeBundleIngestResponse(
|
||||
string BundleId,
|
||||
string GraphHash,
|
||||
string BundleReason,
|
||||
string CasUri,
|
||||
string? DsseCasUri,
|
||||
int EdgeCount,
|
||||
int RevokedCount,
|
||||
bool Quarantined);
|
||||
|
||||
/// <summary>
|
||||
/// Service for ingesting edge-bundle DSSE envelopes.
|
||||
/// </summary>
|
||||
public interface IEdgeBundleIngestionService
|
||||
{
|
||||
/// <summary>
|
||||
/// Ingests an edge bundle from a JSON stream.
|
||||
/// </summary>
|
||||
/// <param name="tenantId">Tenant identifier for isolation.</param>
|
||||
/// <param name="bundleStream">Stream containing the edge-bundle JSON.</param>
|
||||
/// <param name="dsseStream">Optional stream containing the DSSE envelope.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Ingest response with bundle details.</returns>
|
||||
Task<EdgeBundleIngestResponse> IngestAsync(
|
||||
string tenantId,
|
||||
Stream bundleStream,
|
||||
Stream? dsseStream,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets all edge bundles for a graph hash.
|
||||
/// </summary>
|
||||
Task<EdgeBundleDocument[]> GetBundlesForGraphAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Gets revoked edges from all bundles for a graph.
|
||||
/// Returns edges that should be quarantined from scoring.
|
||||
/// </summary>
|
||||
Task<EdgeBundleEdgeDocument[]> GetRevokedEdgesAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if an edge is revoked for the given graph.
|
||||
/// </summary>
|
||||
Task<bool> IsEdgeRevokedAsync(
|
||||
string tenantId,
|
||||
string graphHash,
|
||||
string fromId,
|
||||
string toId,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -1,10 +1,66 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
public interface IRuntimeFactsIngestionService
|
||||
{
|
||||
/// <summary>
|
||||
/// Ingests runtime facts from a structured request.
|
||||
/// </summary>
|
||||
Task<RuntimeFactsIngestResponse> IngestAsync(RuntimeFactsIngestRequest request, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Ingests runtime facts from a raw NDJSON/gzip stream, stores in CAS, and processes.
|
||||
/// </summary>
|
||||
/// <param name="tenantId">Tenant identifier for tenant isolation.</param>
|
||||
/// <param name="content">The NDJSON or gzip compressed stream of runtime fact events.</param>
|
||||
/// <param name="contentType">Content type (application/x-ndjson or application/gzip).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Batch ingestion response with CAS reference.</returns>
|
||||
Task<RuntimeFactsBatchIngestResponse> IngestBatchAsync(
|
||||
string tenantId,
|
||||
Stream content,
|
||||
string contentType,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Response from batch ingestion with CAS storage.
|
||||
/// </summary>
|
||||
public sealed record RuntimeFactsBatchIngestResponse
|
||||
{
|
||||
/// <summary>
|
||||
/// CAS URI for the stored batch artifact.
|
||||
/// </summary>
|
||||
public required string CasUri { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// BLAKE3 hash of the batch artifact.
|
||||
/// </summary>
|
||||
public required string BatchHash { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of fact documents processed.
|
||||
/// </summary>
|
||||
public int ProcessedCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total events ingested.
|
||||
/// </summary>
|
||||
public int TotalEvents { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total hit count across all events.
|
||||
/// </summary>
|
||||
public long TotalHitCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Subject keys affected.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> SubjectKeys { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Timestamp of ingestion.
|
||||
/// </summary>
|
||||
public DateTimeOffset StoredAt { get; init; }
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Cryptography;
|
||||
using StellaOps.Signals.Models;
|
||||
using StellaOps.Signals.Persistence;
|
||||
using StellaOps.Signals.Storage;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
@@ -18,6 +18,8 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
private readonly IEventsPublisher eventsPublisher;
|
||||
private readonly IReachabilityScoringService scoringService;
|
||||
private readonly IRuntimeFactsProvenanceNormalizer provenanceNormalizer;
|
||||
private readonly IRuntimeFactsArtifactStore? artifactStore;
|
||||
private readonly ICryptoHash? cryptoHash;
|
||||
private readonly ILogger<RuntimeFactsIngestionService> logger;
|
||||
|
||||
public RuntimeFactsIngestionService(
|
||||
@@ -27,7 +29,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
IEventsPublisher eventsPublisher,
|
||||
IReachabilityScoringService scoringService,
|
||||
IRuntimeFactsProvenanceNormalizer provenanceNormalizer,
|
||||
ILogger<RuntimeFactsIngestionService> logger)
|
||||
ILogger<RuntimeFactsIngestionService> logger,
|
||||
IRuntimeFactsArtifactStore? artifactStore = null,
|
||||
ICryptoHash? cryptoHash = null)
|
||||
{
|
||||
this.factRepository = factRepository ?? throw new ArgumentNullException(nameof(factRepository));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
@@ -35,6 +39,8 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
this.eventsPublisher = eventsPublisher ?? throw new ArgumentNullException(nameof(eventsPublisher));
|
||||
this.scoringService = scoringService ?? throw new ArgumentNullException(nameof(scoringService));
|
||||
this.provenanceNormalizer = provenanceNormalizer ?? throw new ArgumentNullException(nameof(provenanceNormalizer));
|
||||
this.artifactStore = artifactStore;
|
||||
this.cryptoHash = cryptoHash;
|
||||
this.logger = logger ?? NullLogger<RuntimeFactsIngestionService>.Instance;
|
||||
}
|
||||
|
||||
@@ -96,6 +102,216 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<RuntimeFactsBatchIngestResponse> IngestBatchAsync(
|
||||
string tenantId,
|
||||
Stream content,
|
||||
string contentType,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var storedAt = timeProvider.GetUtcNow();
|
||||
var subjectKeys = new HashSet<string>(StringComparer.Ordinal);
|
||||
var processedCount = 0;
|
||||
var totalEvents = 0;
|
||||
long totalHitCount = 0;
|
||||
|
||||
// Buffer the content for hashing and parsing
|
||||
using var buffer = new MemoryStream();
|
||||
await content.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
|
||||
// Compute BLAKE3 hash
|
||||
string batchHash;
|
||||
if (cryptoHash != null)
|
||||
{
|
||||
batchHash = "blake3:" + await cryptoHash.ComputeHashHexAsync(buffer, "BLAKE3-256", cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fallback: generate a deterministic hash based on content length and timestamp
|
||||
batchHash = $"blake3:{storedAt.ToUnixTimeMilliseconds():x16}{buffer.Length:x16}";
|
||||
}
|
||||
|
||||
// Store to CAS if artifact store is available
|
||||
StoredRuntimeFactsArtifact? storedArtifact = null;
|
||||
if (artifactStore != null)
|
||||
{
|
||||
var fileName = contentType.Contains("gzip", StringComparison.OrdinalIgnoreCase)
|
||||
? "runtime-facts.ndjson.gz"
|
||||
: "runtime-facts.ndjson";
|
||||
|
||||
var saveRequest = new RuntimeFactsArtifactSaveRequest(
|
||||
TenantId: tenantId,
|
||||
SubjectKey: string.Empty, // Will be populated after parsing
|
||||
Hash: batchHash.Replace("blake3:", string.Empty),
|
||||
ContentType: contentType,
|
||||
FileName: fileName,
|
||||
BatchSize: buffer.Length,
|
||||
ProvenanceSource: "runtime-facts-batch");
|
||||
|
||||
storedArtifact = await artifactStore.SaveAsync(saveRequest, buffer, cancellationToken).ConfigureAwait(false);
|
||||
buffer.Position = 0;
|
||||
}
|
||||
|
||||
// Decompress if gzip
|
||||
Stream parseStream;
|
||||
if (contentType.Contains("gzip", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var decompressed = new MemoryStream();
|
||||
await using (var gzip = new GZipStream(buffer, CompressionMode.Decompress, leaveOpen: true))
|
||||
{
|
||||
await gzip.CopyToAsync(decompressed, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
decompressed.Position = 0;
|
||||
parseStream = decompressed;
|
||||
}
|
||||
else
|
||||
{
|
||||
parseStream = buffer;
|
||||
}
|
||||
|
||||
// Parse NDJSON and group by subject
|
||||
var requestsBySubject = new Dictionary<string, RuntimeFactsIngestRequest>(StringComparer.Ordinal);
|
||||
using var reader = new StreamReader(parseStream, leaveOpen: true);
|
||||
|
||||
while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var evt = JsonSerializer.Deserialize<RuntimeFactsBatchEvent>(line, JsonOptions);
|
||||
if (evt is null || string.IsNullOrWhiteSpace(evt.SymbolId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var subjectKey = evt.Subject?.ToSubjectKey() ?? evt.CallgraphId ?? "unknown";
|
||||
if (!requestsBySubject.TryGetValue(subjectKey, out var request))
|
||||
{
|
||||
request = new RuntimeFactsIngestRequest
|
||||
{
|
||||
Subject = evt.Subject ?? new ReachabilitySubject { ScanId = subjectKey },
|
||||
CallgraphId = evt.CallgraphId ?? subjectKey,
|
||||
Events = new List<RuntimeFactEvent>(),
|
||||
Metadata = new Dictionary<string, string?>(StringComparer.Ordinal)
|
||||
{
|
||||
["batch.hash"] = batchHash,
|
||||
["batch.cas_uri"] = storedArtifact?.CasUri,
|
||||
["tenant_id"] = tenantId
|
||||
}
|
||||
};
|
||||
requestsBySubject[subjectKey] = request;
|
||||
}
|
||||
|
||||
((List<RuntimeFactEvent>)request.Events).Add(new RuntimeFactEvent
|
||||
{
|
||||
SymbolId = evt.SymbolId,
|
||||
CodeId = evt.CodeId,
|
||||
SymbolDigest = evt.SymbolDigest,
|
||||
Purl = evt.Purl,
|
||||
BuildId = evt.BuildId,
|
||||
LoaderBase = evt.LoaderBase,
|
||||
ProcessId = evt.ProcessId,
|
||||
ProcessName = evt.ProcessName,
|
||||
SocketAddress = evt.SocketAddress,
|
||||
ContainerId = evt.ContainerId,
|
||||
EvidenceUri = evt.EvidenceUri,
|
||||
HitCount = Math.Max(evt.HitCount, 1),
|
||||
ObservedAt = evt.ObservedAt,
|
||||
Metadata = evt.Metadata
|
||||
});
|
||||
|
||||
totalEvents++;
|
||||
totalHitCount += Math.Max(evt.HitCount, 1);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Failed to parse NDJSON line in batch ingestion.");
|
||||
}
|
||||
}
|
||||
|
||||
// Process each subject's request
|
||||
foreach (var (subjectKey, request) in requestsBySubject)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await IngestAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Update the fact document with batch reference
|
||||
var existing = await factRepository.GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
if (existing != null && storedArtifact != null)
|
||||
{
|
||||
existing.RuntimeFactsBatchUri = storedArtifact.CasUri;
|
||||
existing.RuntimeFactsBatchHash = batchHash;
|
||||
await factRepository.UpsertAsync(existing, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
subjectKeys.Add(subjectKey);
|
||||
processedCount++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to ingest batch for subject {SubjectKey}.", subjectKey);
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Batch ingestion completed: {ProcessedCount} subjects, {TotalEvents} events, {TotalHitCount} hits (hash={BatchHash}, tenant={TenantId}).",
|
||||
processedCount,
|
||||
totalEvents,
|
||||
totalHitCount,
|
||||
batchHash,
|
||||
tenantId);
|
||||
|
||||
return new RuntimeFactsBatchIngestResponse
|
||||
{
|
||||
CasUri = storedArtifact?.CasUri ?? $"cas://reachability/runtime-facts/{batchHash.Replace("blake3:", string.Empty)}",
|
||||
BatchHash = batchHash,
|
||||
ProcessedCount = processedCount,
|
||||
TotalEvents = totalEvents,
|
||||
TotalHitCount = totalHitCount,
|
||||
SubjectKeys = subjectKeys.ToList(),
|
||||
StoredAt = storedAt
|
||||
};
|
||||
}
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// NDJSON batch event structure for runtime facts.
|
||||
/// </summary>
|
||||
private sealed class RuntimeFactsBatchEvent
|
||||
{
|
||||
public string? SymbolId { get; set; }
|
||||
public string? CodeId { get; set; }
|
||||
public string? SymbolDigest { get; set; }
|
||||
public string? Purl { get; set; }
|
||||
public string? BuildId { get; set; }
|
||||
public string? LoaderBase { get; set; }
|
||||
public int? ProcessId { get; set; }
|
||||
public string? ProcessName { get; set; }
|
||||
public string? SocketAddress { get; set; }
|
||||
public string? ContainerId { get; set; }
|
||||
public string? EvidenceUri { get; set; }
|
||||
public int HitCount { get; set; } = 1;
|
||||
public DateTimeOffset? ObservedAt { get; set; }
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
public ReachabilitySubject? Subject { get; set; }
|
||||
public string? CallgraphId { get; set; }
|
||||
}
|
||||
|
||||
private static void ValidateRequest(RuntimeFactsIngestRequest request)
|
||||
{
|
||||
if (request.Subject is null)
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Stores runtime-facts batch artifacts on the local filesystem.
|
||||
/// CAS paths: cas://reachability/runtime-facts/{hash}
|
||||
/// </summary>
|
||||
internal sealed class FileSystemRuntimeFactsArtifactStore : IRuntimeFactsArtifactStore
|
||||
{
|
||||
private const string DefaultFileName = "runtime-facts.ndjson";
|
||||
|
||||
private readonly SignalsArtifactStorageOptions _storageOptions;
|
||||
private readonly ILogger<FileSystemRuntimeFactsArtifactStore> _logger;
|
||||
|
||||
public FileSystemRuntimeFactsArtifactStore(
|
||||
IOptions<SignalsOptions> options,
|
||||
ILogger<FileSystemRuntimeFactsArtifactStore> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_storageOptions = options.Value.Storage;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<StoredRuntimeFactsArtifact> SaveAsync(
|
||||
RuntimeFactsArtifactSaveRequest request,
|
||||
Stream content,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var hash = NormalizeHash(request.Hash);
|
||||
if (string.IsNullOrWhiteSpace(hash))
|
||||
{
|
||||
throw new InvalidOperationException("Runtime-facts artifact hash is required for CAS storage.");
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(hash);
|
||||
Directory.CreateDirectory(casDirectory);
|
||||
|
||||
var fileName = SanitizeFileName(string.IsNullOrWhiteSpace(request.FileName) ? DefaultFileName : request.FileName);
|
||||
var destinationPath = Path.Combine(casDirectory, fileName);
|
||||
|
||||
await using (var fileStream = File.Create(destinationPath))
|
||||
{
|
||||
await content.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var fileInfo = new FileInfo(destinationPath);
|
||||
var casUri = $"cas://reachability/runtime-facts/{hash}";
|
||||
|
||||
_logger.LogInformation(
|
||||
"Stored runtime-facts artifact at {Path} (length={Length}, hash={Hash}, tenant={TenantId}).",
|
||||
destinationPath,
|
||||
fileInfo.Length,
|
||||
hash,
|
||||
request.TenantId);
|
||||
|
||||
return new StoredRuntimeFactsArtifact(
|
||||
Path.GetRelativePath(_storageOptions.RootPath, destinationPath),
|
||||
fileInfo.Length,
|
||||
hash,
|
||||
request.ContentType,
|
||||
casUri);
|
||||
}
|
||||
|
||||
public Task<Stream?> GetAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
var filePath = Path.Combine(casDirectory, DefaultFileName);
|
||||
|
||||
// Also check for gzip variant
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
filePath = Path.Combine(casDirectory, "runtime-facts.ndjson.gz");
|
||||
}
|
||||
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
_logger.LogDebug("Runtime-facts artifact {Hash} not found at {Path}.", normalizedHash, filePath);
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
var content = new MemoryStream();
|
||||
using (var fileStream = File.OpenRead(filePath))
|
||||
{
|
||||
fileStream.CopyTo(content);
|
||||
}
|
||||
|
||||
content.Position = 0;
|
||||
_logger.LogDebug("Retrieved runtime-facts artifact {Hash} from {Path}.", normalizedHash, filePath);
|
||||
return Task.FromResult<Stream?>(content);
|
||||
}
|
||||
|
||||
public Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
var defaultPath = Path.Combine(casDirectory, DefaultFileName);
|
||||
var gzipPath = Path.Combine(casDirectory, "runtime-facts.ndjson.gz");
|
||||
var exists = File.Exists(defaultPath) || File.Exists(gzipPath);
|
||||
|
||||
_logger.LogDebug("Runtime-facts artifact {Hash} exists={Exists}.", normalizedHash, exists);
|
||||
return Task.FromResult(exists);
|
||||
}
|
||||
|
||||
public Task<bool> DeleteAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
if (!Directory.Exists(casDirectory))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(casDirectory, recursive: true);
|
||||
_logger.LogInformation("Deleted runtime-facts artifact {Hash}.", normalizedHash);
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to delete runtime-facts artifact {Hash}.", normalizedHash);
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
}
|
||||
|
||||
private string GetCasDirectory(string hash)
|
||||
{
|
||||
var prefix = hash.Length >= 2 ? hash[..2] : hash;
|
||||
return Path.Combine(_storageOptions.RootPath, "cas", "reachability", "runtime-facts", prefix, hash);
|
||||
}
|
||||
|
||||
private static string? NormalizeHash(string? hash)
|
||||
=> hash?.Trim().ToLowerInvariant();
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
=> string.Join('_', value.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)).ToLowerInvariant();
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Persists and retrieves runtime-facts batch artifacts from content-addressable storage.
|
||||
/// CAS paths follow: cas://reachability/runtime-facts/{hash}
|
||||
/// </summary>
|
||||
public interface IRuntimeFactsArtifactStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Stores a runtime-facts batch artifact.
|
||||
/// </summary>
|
||||
/// <param name="request">Metadata about the artifact to store.</param>
|
||||
/// <param name="content">The artifact content stream (NDJSON or gzip compressed).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Information about the stored artifact including CAS URI.</returns>
|
||||
Task<StoredRuntimeFactsArtifact> SaveAsync(
|
||||
RuntimeFactsArtifactSaveRequest request,
|
||||
Stream content,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a runtime-facts artifact by its BLAKE3 hash.
|
||||
/// </summary>
|
||||
/// <param name="hash">The BLAKE3 hash of the artifact (blake3:{hex}).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The artifact content stream, or null if not found.</returns>
|
||||
Task<Stream?> GetAsync(string hash, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a runtime-facts artifact exists.
|
||||
/// </summary>
|
||||
/// <param name="hash">The BLAKE3 hash of the artifact.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>True if the artifact exists.</returns>
|
||||
Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a runtime-facts artifact if it exists.
|
||||
/// </summary>
|
||||
/// <param name="hash">The BLAKE3 hash of the artifact.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>True if the artifact was deleted, false if it did not exist.</returns>
|
||||
Task<bool> DeleteAsync(string hash, CancellationToken cancellationToken = default);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace StellaOps.Signals.Storage.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Context required to persist a runtime-facts artifact batch.
|
||||
/// </summary>
|
||||
public sealed record RuntimeFactsArtifactSaveRequest(
|
||||
string TenantId,
|
||||
string SubjectKey,
|
||||
string Hash,
|
||||
string ContentType,
|
||||
string FileName,
|
||||
long BatchSize,
|
||||
string? ProvenanceSource);
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace StellaOps.Signals.Storage.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Result returned after storing a runtime-facts artifact.
|
||||
/// </summary>
|
||||
public sealed record StoredRuntimeFactsArtifact(
|
||||
string Path,
|
||||
long Length,
|
||||
string Hash,
|
||||
string ContentType,
|
||||
string CasUri);
|
||||
Reference in New Issue
Block a user