up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-12-14 15:50:38 +02:00
parent f1a39c4ce3
commit 233873f620
249 changed files with 29746 additions and 154 deletions

View File

@@ -0,0 +1,179 @@
using System;
using System.Collections.Generic;
namespace StellaOps.Signals.Models;
/// <summary>
/// Edge bundle document for storing ingested edge bundles.
/// </summary>
public sealed class EdgeBundleDocument
{
public string Id { get; set; } = Guid.NewGuid().ToString("N");
/// <summary>
/// Bundle identifier from the DSSE envelope.
/// </summary>
public string BundleId { get; set; } = string.Empty;
/// <summary>
/// Graph hash this bundle is associated with.
/// </summary>
public string GraphHash { get; set; } = string.Empty;
/// <summary>
/// Tenant identifier for isolation.
/// </summary>
public string TenantId { get; set; } = string.Empty;
/// <summary>
/// Reason for this bundle (RuntimeHits, InitArray, ThirdParty, Contested, Revoked, etc.).
/// </summary>
public string BundleReason { get; set; } = string.Empty;
/// <summary>
/// Custom reason description if BundleReason is Custom.
/// </summary>
public string? CustomReason { get; set; }
/// <summary>
/// Edges in this bundle.
/// </summary>
public List<EdgeBundleEdgeDocument> Edges { get; set; } = new();
/// <summary>
/// Content hash of the bundle (sha256:...).
/// </summary>
public string ContentHash { get; set; } = string.Empty;
/// <summary>
/// DSSE envelope digest.
/// </summary>
public string? DsseDigest { get; set; }
/// <summary>
/// CAS URI for the bundle JSON.
/// </summary>
public string? CasUri { get; set; }
/// <summary>
/// CAS URI for the DSSE envelope.
/// </summary>
public string? DsseCasUri { get; set; }
/// <summary>
/// Whether this bundle has been verified.
/// </summary>
public bool Verified { get; set; }
/// <summary>
/// Rekor log index if published.
/// </summary>
public long? RekorLogIndex { get; set; }
/// <summary>
/// Count of revoked edges in this bundle.
/// </summary>
public int RevokedCount { get; set; }
/// <summary>
/// When the bundle was ingested.
/// </summary>
public DateTimeOffset IngestedAt { get; set; }
/// <summary>
/// When the bundle was generated.
/// </summary>
public DateTimeOffset GeneratedAt { get; set; }
}
/// <summary>
/// Individual edge within an edge bundle document.
/// </summary>
public sealed class EdgeBundleEdgeDocument
{
/// <summary>
/// Source function/method ID.
/// </summary>
public string From { get; set; } = string.Empty;
/// <summary>
/// Target function/method ID.
/// </summary>
public string To { get; set; } = string.Empty;
/// <summary>
/// Edge kind (call, callvirt, invokestatic, etc.).
/// </summary>
public string Kind { get; set; } = "call";
/// <summary>
/// Reason for inclusion in this bundle.
/// </summary>
public string Reason { get; set; } = string.Empty;
/// <summary>
/// Whether this edge is revoked (patched/removed).
/// </summary>
public bool Revoked { get; set; }
/// <summary>
/// Confidence level (0.0-1.0).
/// </summary>
public double Confidence { get; set; }
/// <summary>
/// Package URL of the target.
/// </summary>
public string? Purl { get; set; }
/// <summary>
/// Symbol digest of the target.
/// </summary>
public string? SymbolDigest { get; set; }
/// <summary>
/// Evidence URI for this edge.
/// </summary>
public string? Evidence { get; set; }
}
/// <summary>
/// Reference to an edge bundle attached to a reachability fact.
/// </summary>
public sealed class EdgeBundleReference
{
/// <summary>
/// Bundle identifier.
/// </summary>
public string BundleId { get; set; } = string.Empty;
/// <summary>
/// Bundle reason.
/// </summary>
public string BundleReason { get; set; } = string.Empty;
/// <summary>
/// CAS URI for the bundle.
/// </summary>
public string? CasUri { get; set; }
/// <summary>
/// DSSE CAS URI.
/// </summary>
public string? DsseCasUri { get; set; }
/// <summary>
/// Number of edges in the bundle.
/// </summary>
public int EdgeCount { get; set; }
/// <summary>
/// Number of revoked edges.
/// </summary>
public int RevokedCount { get; set; }
/// <summary>
/// Whether the bundle has been verified.
/// </summary>
public bool Verified { get; set; }
}

View File

@@ -0,0 +1,232 @@
using System.Text.Json.Serialization;
namespace StellaOps.Signals.Models;
/// <summary>
/// Document representing a proc snapshot for Java/.NET/PHP runtime parity.
/// Captures runtime-observed classpath, loaded assemblies, and autoload paths.
/// </summary>
public sealed class ProcSnapshotDocument
{
/// <summary>
/// Unique identifier for this snapshot (format: {tenant}/{image_digest}/{snapshot_hash}).
/// </summary>
public required string Id { get; init; }
/// <summary>
/// Tenant identifier for multi-tenancy isolation.
/// </summary>
public required string Tenant { get; init; }
/// <summary>
/// Image digest of the container this snapshot was captured from.
/// </summary>
[JsonPropertyName("imageDigest")]
public required string ImageDigest { get; init; }
/// <summary>
/// Node identifier where the snapshot was captured.
/// </summary>
public string? Node { get; init; }
/// <summary>
/// Container ID where the process was running.
/// </summary>
public string? ContainerId { get; init; }
/// <summary>
/// Process ID at capture time.
/// </summary>
public int Pid { get; init; }
/// <summary>
/// Process entrypoint command.
/// </summary>
public string? Entrypoint { get; init; }
/// <summary>
/// Runtime type: java, dotnet, php.
/// </summary>
public required string RuntimeType { get; init; }
/// <summary>
/// Runtime version (e.g., "17.0.2", "8.0.0", "8.2.0").
/// </summary>
public string? RuntimeVersion { get; init; }
/// <summary>
/// Java classpath entries (jar paths, directories).
/// </summary>
[JsonPropertyName("classpath")]
public IReadOnlyList<ClasspathEntry> Classpath { get; init; } = Array.Empty<ClasspathEntry>();
/// <summary>
/// .NET loaded assemblies with RID-graph resolution.
/// </summary>
[JsonPropertyName("loadedAssemblies")]
public IReadOnlyList<LoadedAssemblyEntry> LoadedAssemblies { get; init; } = Array.Empty<LoadedAssemblyEntry>();
/// <summary>
/// PHP autoload paths from composer autoloader.
/// </summary>
[JsonPropertyName("autoloadPaths")]
public IReadOnlyList<AutoloadPathEntry> AutoloadPaths { get; init; } = Array.Empty<AutoloadPathEntry>();
/// <summary>
/// Timestamp when the snapshot was captured.
/// </summary>
public DateTimeOffset CapturedAt { get; init; }
/// <summary>
/// Timestamp when the snapshot was stored.
/// </summary>
public DateTimeOffset StoredAt { get; init; }
/// <summary>
/// Expiration timestamp for TTL-based cleanup.
/// </summary>
public DateTimeOffset? ExpiresAt { get; init; }
/// <summary>
/// Additional annotations/metadata.
/// </summary>
public IReadOnlyDictionary<string, string>? Annotations { get; init; }
}
/// <summary>
/// Java classpath entry representing a JAR or directory.
/// </summary>
public sealed class ClasspathEntry
{
/// <summary>
/// Path to the JAR file or classpath directory.
/// </summary>
public required string Path { get; init; }
/// <summary>
/// Type: jar, directory, jmod.
/// </summary>
public string? Type { get; init; }
/// <summary>
/// SHA-256 hash of the JAR file (null for directories).
/// </summary>
public string? Sha256 { get; init; }
/// <summary>
/// Maven coordinate if resolvable (e.g., "org.springframework:spring-core:5.3.20").
/// </summary>
public string? MavenCoordinate { get; init; }
/// <summary>
/// Package URL (PURL) if resolvable.
/// </summary>
public string? Purl { get; init; }
/// <summary>
/// Size in bytes.
/// </summary>
public long? SizeBytes { get; init; }
}
/// <summary>
/// .NET loaded assembly entry with RID-graph context.
/// </summary>
public sealed class LoadedAssemblyEntry
{
/// <summary>
/// Assembly name (e.g., "System.Text.Json").
/// </summary>
public required string Name { get; init; }
/// <summary>
/// Assembly version (e.g., "8.0.0.0").
/// </summary>
public string? Version { get; init; }
/// <summary>
/// Full path to the loaded DLL.
/// </summary>
public required string Path { get; init; }
/// <summary>
/// SHA-256 hash of the assembly file.
/// </summary>
public string? Sha256 { get; init; }
/// <summary>
/// NuGet package ID if resolvable.
/// </summary>
public string? NuGetPackage { get; init; }
/// <summary>
/// NuGet package version if resolvable.
/// </summary>
public string? NuGetVersion { get; init; }
/// <summary>
/// Package URL (PURL) if resolvable.
/// </summary>
public string? Purl { get; init; }
/// <summary>
/// Runtime identifier (RID) for platform-specific assemblies.
/// </summary>
public string? Rid { get; init; }
/// <summary>
/// Whether this assembly was loaded from the shared framework.
/// </summary>
public bool? IsFrameworkAssembly { get; init; }
/// <summary>
/// Source from deps.json resolution: compile, runtime, native.
/// </summary>
public string? DepsSource { get; init; }
}
/// <summary>
/// PHP autoload path entry from Composer.
/// </summary>
public sealed class AutoloadPathEntry
{
/// <summary>
/// Namespace prefix (PSR-4) or class name (classmap).
/// </summary>
public string? Namespace { get; init; }
/// <summary>
/// Autoload type: psr-4, psr-0, classmap, files.
/// </summary>
public required string Type { get; init; }
/// <summary>
/// Path to the autoloaded file or directory.
/// </summary>
public required string Path { get; init; }
/// <summary>
/// Composer package name if resolvable.
/// </summary>
public string? ComposerPackage { get; init; }
/// <summary>
/// Composer package version if resolvable.
/// </summary>
public string? ComposerVersion { get; init; }
/// <summary>
/// Package URL (PURL) if resolvable.
/// </summary>
public string? Purl { get; init; }
}
/// <summary>
/// Known runtime types for proc snapshots.
/// </summary>
public static class ProcSnapshotRuntimeTypes
{
public const string Java = "java";
public const string DotNet = "dotnet";
public const string Php = "php";
}

View File

@@ -17,12 +17,32 @@ public sealed class ReachabilityFactDocument
public List<RuntimeFactDocument>? RuntimeFacts { get; set; }
/// <summary>
/// CAS URI for the runtime-facts batch artifact (cas://reachability/runtime-facts/{hash}).
/// </summary>
public string? RuntimeFactsBatchUri { get; set; }
/// <summary>
/// BLAKE3 hash of the runtime-facts batch artifact.
/// </summary>
public string? RuntimeFactsBatchHash { get; set; }
public Dictionary<string, string?>? Metadata { get; set; }
public ContextFacts? ContextFacts { get; set; }
public UncertaintyDocument? Uncertainty { get; set; }
/// <summary>
/// Edge bundles attached to this graph.
/// </summary>
public List<EdgeBundleReference>? EdgeBundles { get; set; }
/// <summary>
/// Whether any edges are quarantined (revoked) for this fact.
/// </summary>
public bool HasQuarantinedEdges { get; set; }
public double Score { get; set; }
public double RiskScore { get; set; }

View File

@@ -0,0 +1,42 @@
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Persistence;
/// <summary>
/// Repository for persisting and querying proc snapshot documents.
/// </summary>
public interface IProcSnapshotRepository
{
/// <summary>
/// Upsert a proc snapshot document.
/// </summary>
Task<ProcSnapshotDocument> UpsertAsync(ProcSnapshotDocument document, CancellationToken cancellationToken);
/// <summary>
/// Get a proc snapshot by ID.
/// </summary>
Task<ProcSnapshotDocument?> GetByIdAsync(string id, CancellationToken cancellationToken);
/// <summary>
/// Get all proc snapshots for a specific image digest.
/// </summary>
Task<IReadOnlyList<ProcSnapshotDocument>> GetByImageDigestAsync(
string tenant,
string imageDigest,
int limit = 100,
CancellationToken cancellationToken = default);
/// <summary>
/// Get the most recent proc snapshot for an image digest and runtime type.
/// </summary>
Task<ProcSnapshotDocument?> GetLatestAsync(
string tenant,
string imageDigest,
string runtimeType,
CancellationToken cancellationToken);
/// <summary>
/// Delete expired proc snapshots.
/// </summary>
Task<int> DeleteExpiredAsync(CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,95 @@
using System.Collections.Concurrent;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Persistence;
/// <summary>
/// In-memory implementation of <see cref="IProcSnapshotRepository"/> for testing and development.
/// </summary>
public sealed class InMemoryProcSnapshotRepository : IProcSnapshotRepository
{
private readonly ConcurrentDictionary<string, ProcSnapshotDocument> _documents = new(StringComparer.OrdinalIgnoreCase);
private readonly TimeProvider _timeProvider;
public InMemoryProcSnapshotRepository(TimeProvider timeProvider)
{
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
}
public Task<ProcSnapshotDocument> UpsertAsync(ProcSnapshotDocument document, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(document);
_documents[document.Id] = document;
return Task.FromResult(document);
}
public Task<ProcSnapshotDocument?> GetByIdAsync(string id, CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(id);
_documents.TryGetValue(id, out var document);
return Task.FromResult(document);
}
public Task<IReadOnlyList<ProcSnapshotDocument>> GetByImageDigestAsync(
string tenant,
string imageDigest,
int limit = 100,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
var normalizedDigest = imageDigest.ToLowerInvariant();
var results = _documents.Values
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase) &&
string.Equals(d.ImageDigest, normalizedDigest, StringComparison.OrdinalIgnoreCase))
.OrderByDescending(d => d.CapturedAt)
.Take(limit)
.ToList();
return Task.FromResult<IReadOnlyList<ProcSnapshotDocument>>(results);
}
public Task<ProcSnapshotDocument?> GetLatestAsync(
string tenant,
string imageDigest,
string runtimeType,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenant);
ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest);
ArgumentException.ThrowIfNullOrWhiteSpace(runtimeType);
var normalizedDigest = imageDigest.ToLowerInvariant();
var result = _documents.Values
.Where(d => string.Equals(d.Tenant, tenant, StringComparison.OrdinalIgnoreCase) &&
string.Equals(d.ImageDigest, normalizedDigest, StringComparison.OrdinalIgnoreCase) &&
string.Equals(d.RuntimeType, runtimeType, StringComparison.OrdinalIgnoreCase))
.OrderByDescending(d => d.CapturedAt)
.FirstOrDefault();
return Task.FromResult(result);
}
public Task<int> DeleteExpiredAsync(CancellationToken cancellationToken)
{
var now = _timeProvider.GetUtcNow();
var expiredIds = _documents
.Where(kv => kv.Value.ExpiresAt.HasValue && kv.Value.ExpiresAt.Value < now)
.Select(kv => kv.Key)
.ToList();
var deletedCount = 0;
foreach (var id in expiredIds)
{
if (_documents.TryRemove(id, out _))
{
deletedCount++;
}
}
return Task.FromResult(deletedCount);
}
}

View File

@@ -0,0 +1,261 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Signals.Models;
using StellaOps.Signals.Options;
namespace StellaOps.Signals.Services;
/// <summary>
/// Ingests edge-bundle DSSE envelopes, attaches to graph_hash, enforces quarantine for revoked edges.
/// </summary>
public sealed class EdgeBundleIngestionService : IEdgeBundleIngestionService
{
private readonly ILogger<EdgeBundleIngestionService> _logger;
private readonly SignalsOptions _options;
// In-memory storage (in production, would use repository)
private readonly ConcurrentDictionary<string, List<EdgeBundleDocument>> _bundlesByGraphHash = new();
private readonly ConcurrentDictionary<string, HashSet<string>> _revokedEdgeKeys = new();
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
{
PropertyNameCaseInsensitive = true
};
public EdgeBundleIngestionService(
ILogger<EdgeBundleIngestionService> logger,
IOptions<SignalsOptions> options)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
}
public async Task<EdgeBundleIngestResponse> IngestAsync(
string tenantId,
Stream bundleStream,
Stream? dsseStream,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentNullException.ThrowIfNull(bundleStream);
// Parse the bundle JSON
using var bundleMs = new MemoryStream();
await bundleStream.CopyToAsync(bundleMs, cancellationToken).ConfigureAwait(false);
bundleMs.Position = 0;
var bundleJson = await JsonDocument.ParseAsync(bundleMs, cancellationToken: cancellationToken).ConfigureAwait(false);
var root = bundleJson.RootElement;
// Extract bundle fields
var bundleId = GetStringOrDefault(root, "bundleId", $"bundle:{Guid.NewGuid():N}");
var graphHash = GetStringOrDefault(root, "graphHash", string.Empty);
var bundleReason = GetStringOrDefault(root, "bundleReason", "Custom");
var customReason = GetStringOrDefault(root, "customReason", null);
var generatedAtStr = GetStringOrDefault(root, "generatedAt", null);
if (string.IsNullOrWhiteSpace(graphHash))
{
throw new InvalidOperationException("Edge bundle missing required 'graphHash' field");
}
var generatedAt = !string.IsNullOrWhiteSpace(generatedAtStr)
? DateTimeOffset.Parse(generatedAtStr)
: DateTimeOffset.UtcNow;
// Parse edges
var edges = new List<EdgeBundleEdgeDocument>();
var revokedCount = 0;
if (root.TryGetProperty("edges", out var edgesElement) && edgesElement.ValueKind == JsonValueKind.Array)
{
foreach (var edgeEl in edgesElement.EnumerateArray())
{
var edge = new EdgeBundleEdgeDocument
{
From = GetStringOrDefault(edgeEl, "from", string.Empty),
To = GetStringOrDefault(edgeEl, "to", string.Empty),
Kind = GetStringOrDefault(edgeEl, "kind", "call"),
Reason = GetStringOrDefault(edgeEl, "reason", "Unknown"),
Revoked = edgeEl.TryGetProperty("revoked", out var r) && r.GetBoolean(),
Confidence = edgeEl.TryGetProperty("confidence", out var c) ? c.GetDouble() : 0.5,
Purl = GetStringOrDefault(edgeEl, "purl", null),
SymbolDigest = GetStringOrDefault(edgeEl, "symbolDigest", null),
Evidence = GetStringOrDefault(edgeEl, "evidence", null)
};
edges.Add(edge);
if (edge.Revoked)
{
revokedCount++;
}
}
}
// Compute content hash
bundleMs.Position = 0;
var contentHash = ComputeSha256(bundleMs);
// Parse DSSE if provided
string? dsseDigest = null;
if (dsseStream is not null)
{
using var dsseMs = new MemoryStream();
await dsseStream.CopyToAsync(dsseMs, cancellationToken).ConfigureAwait(false);
dsseMs.Position = 0;
dsseDigest = $"sha256:{ComputeSha256(dsseMs)}";
}
// Build CAS URIs
var graphHashDigest = ExtractHashDigest(graphHash);
var casUri = $"cas://reachability/edges/{graphHashDigest}/{bundleId}";
var dsseCasUri = dsseStream is not null ? $"{casUri}.dsse" : null;
// Create document
var document = new EdgeBundleDocument
{
BundleId = bundleId,
GraphHash = graphHash,
TenantId = tenantId,
BundleReason = bundleReason,
CustomReason = customReason,
Edges = edges,
ContentHash = $"sha256:{contentHash}",
DsseDigest = dsseDigest,
CasUri = casUri,
DsseCasUri = dsseCasUri,
Verified = dsseStream is not null, // Simple verification - in production would verify signature
RevokedCount = revokedCount,
IngestedAt = DateTimeOffset.UtcNow,
GeneratedAt = generatedAt
};
// Store document
var storageKey = $"{tenantId}:{graphHash}";
_bundlesByGraphHash.AddOrUpdate(
storageKey,
_ => new List<EdgeBundleDocument> { document },
(_, list) =>
{
// Remove existing bundle with same ID
list.RemoveAll(b => b.BundleId == bundleId);
list.Add(document);
return list;
});
// Update revoked edge index for quarantine enforcement
if (revokedCount > 0)
{
var revokedEdges = edges.Where(e => e.Revoked).Select(e => $"{e.From}>{e.To}").ToHashSet();
_revokedEdgeKeys.AddOrUpdate(
storageKey,
_ => revokedEdges,
(_, existing) =>
{
foreach (var key in revokedEdges)
{
existing.Add(key);
}
return existing;
});
}
var quarantined = revokedCount > 0;
_logger.LogInformation(
"Ingested edge bundle {BundleId} for graph {GraphHash} with {EdgeCount} edges ({RevokedCount} revoked, quarantine={Quarantined})",
bundleId, graphHash, edges.Count, revokedCount, quarantined);
return new EdgeBundleIngestResponse(
bundleId,
graphHash,
bundleReason,
casUri,
dsseCasUri,
edges.Count,
revokedCount,
quarantined);
}
public Task<EdgeBundleDocument[]> GetBundlesForGraphAsync(
string tenantId,
string graphHash,
CancellationToken cancellationToken = default)
{
var key = $"{tenantId}:{graphHash}";
if (_bundlesByGraphHash.TryGetValue(key, out var bundles))
{
return Task.FromResult(bundles.ToArray());
}
return Task.FromResult(Array.Empty<EdgeBundleDocument>());
}
public Task<EdgeBundleEdgeDocument[]> GetRevokedEdgesAsync(
string tenantId,
string graphHash,
CancellationToken cancellationToken = default)
{
var key = $"{tenantId}:{graphHash}";
if (_bundlesByGraphHash.TryGetValue(key, out var bundles))
{
var revoked = bundles
.SelectMany(b => b.Edges)
.Where(e => e.Revoked)
.ToArray();
return Task.FromResult(revoked);
}
return Task.FromResult(Array.Empty<EdgeBundleEdgeDocument>());
}
public Task<bool> IsEdgeRevokedAsync(
string tenantId,
string graphHash,
string fromId,
string toId,
CancellationToken cancellationToken = default)
{
var key = $"{tenantId}:{graphHash}";
if (_revokedEdgeKeys.TryGetValue(key, out var revokedKeys))
{
var edgeKey = $"{fromId}>{toId}";
return Task.FromResult(revokedKeys.Contains(edgeKey));
}
return Task.FromResult(false);
}
private static string GetStringOrDefault(JsonElement element, string propertyName, string? defaultValue)
{
if (element.TryGetProperty(propertyName, out var prop) && prop.ValueKind == JsonValueKind.String)
{
return prop.GetString() ?? defaultValue ?? string.Empty;
}
return defaultValue ?? string.Empty;
}
private static string ComputeSha256(Stream stream)
{
using var sha = SHA256.Create();
var hash = sha.ComputeHash(stream);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static string ExtractHashDigest(string prefixedHash)
{
var colonIndex = prefixedHash.IndexOf(':');
return colonIndex >= 0 ? prefixedHash[(colonIndex + 1)..] : prefixedHash;
}
}

View File

@@ -0,0 +1,66 @@
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Services;
/// <summary>
/// Response from edge bundle ingestion.
/// </summary>
public sealed record EdgeBundleIngestResponse(
string BundleId,
string GraphHash,
string BundleReason,
string CasUri,
string? DsseCasUri,
int EdgeCount,
int RevokedCount,
bool Quarantined);
/// <summary>
/// Service for ingesting edge-bundle DSSE envelopes.
/// </summary>
public interface IEdgeBundleIngestionService
{
/// <summary>
/// Ingests an edge bundle from a JSON stream.
/// </summary>
/// <param name="tenantId">Tenant identifier for isolation.</param>
/// <param name="bundleStream">Stream containing the edge-bundle JSON.</param>
/// <param name="dsseStream">Optional stream containing the DSSE envelope.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Ingest response with bundle details.</returns>
Task<EdgeBundleIngestResponse> IngestAsync(
string tenantId,
Stream bundleStream,
Stream? dsseStream,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets all edge bundles for a graph hash.
/// </summary>
Task<EdgeBundleDocument[]> GetBundlesForGraphAsync(
string tenantId,
string graphHash,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets revoked edges from all bundles for a graph.
/// Returns edges that should be quarantined from scoring.
/// </summary>
Task<EdgeBundleEdgeDocument[]> GetRevokedEdgesAsync(
string tenantId,
string graphHash,
CancellationToken cancellationToken = default);
/// <summary>
/// Checks if an edge is revoked for the given graph.
/// </summary>
Task<bool> IsEdgeRevokedAsync(
string tenantId,
string graphHash,
string fromId,
string toId,
CancellationToken cancellationToken = default);
}

View File

@@ -1,10 +1,66 @@
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Signals.Models;
namespace StellaOps.Signals.Services;
public interface IRuntimeFactsIngestionService
{
/// <summary>
/// Ingests runtime facts from a structured request.
/// </summary>
Task<RuntimeFactsIngestResponse> IngestAsync(RuntimeFactsIngestRequest request, CancellationToken cancellationToken);
/// <summary>
/// Ingests runtime facts from a raw NDJSON/gzip stream, stores in CAS, and processes.
/// </summary>
/// <param name="tenantId">Tenant identifier for tenant isolation.</param>
/// <param name="content">The NDJSON or gzip compressed stream of runtime fact events.</param>
/// <param name="contentType">Content type (application/x-ndjson or application/gzip).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Batch ingestion response with CAS reference.</returns>
Task<RuntimeFactsBatchIngestResponse> IngestBatchAsync(
string tenantId,
Stream content,
string contentType,
CancellationToken cancellationToken);
}
/// <summary>
/// Response from batch ingestion with CAS storage.
/// </summary>
public sealed record RuntimeFactsBatchIngestResponse
{
/// <summary>
/// CAS URI for the stored batch artifact.
/// </summary>
public required string CasUri { get; init; }
/// <summary>
/// BLAKE3 hash of the batch artifact.
/// </summary>
public required string BatchHash { get; init; }
/// <summary>
/// Number of fact documents processed.
/// </summary>
public int ProcessedCount { get; init; }
/// <summary>
/// Total events ingested.
/// </summary>
public int TotalEvents { get; init; }
/// <summary>
/// Total hit count across all events.
/// </summary>
public long TotalHitCount { get; init; }
/// <summary>
/// Subject keys affected.
/// </summary>
public IReadOnlyList<string> SubjectKeys { get; init; } = [];
/// <summary>
/// Timestamp of ingestion.
/// </summary>
public DateTimeOffset StoredAt { get; init; }
}

View File

@@ -1,12 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.IO.Compression;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Cryptography;
using StellaOps.Signals.Models;
using StellaOps.Signals.Persistence;
using StellaOps.Signals.Storage;
using StellaOps.Signals.Storage.Models;
namespace StellaOps.Signals.Services;
@@ -18,6 +18,8 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
private readonly IEventsPublisher eventsPublisher;
private readonly IReachabilityScoringService scoringService;
private readonly IRuntimeFactsProvenanceNormalizer provenanceNormalizer;
private readonly IRuntimeFactsArtifactStore? artifactStore;
private readonly ICryptoHash? cryptoHash;
private readonly ILogger<RuntimeFactsIngestionService> logger;
public RuntimeFactsIngestionService(
@@ -27,7 +29,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
IEventsPublisher eventsPublisher,
IReachabilityScoringService scoringService,
IRuntimeFactsProvenanceNormalizer provenanceNormalizer,
ILogger<RuntimeFactsIngestionService> logger)
ILogger<RuntimeFactsIngestionService> logger,
IRuntimeFactsArtifactStore? artifactStore = null,
ICryptoHash? cryptoHash = null)
{
this.factRepository = factRepository ?? throw new ArgumentNullException(nameof(factRepository));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
@@ -35,6 +39,8 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
this.eventsPublisher = eventsPublisher ?? throw new ArgumentNullException(nameof(eventsPublisher));
this.scoringService = scoringService ?? throw new ArgumentNullException(nameof(scoringService));
this.provenanceNormalizer = provenanceNormalizer ?? throw new ArgumentNullException(nameof(provenanceNormalizer));
this.artifactStore = artifactStore;
this.cryptoHash = cryptoHash;
this.logger = logger ?? NullLogger<RuntimeFactsIngestionService>.Instance;
}
@@ -96,6 +102,216 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
};
}
public async Task<RuntimeFactsBatchIngestResponse> IngestBatchAsync(
string tenantId,
Stream content,
string contentType,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentNullException.ThrowIfNull(content);
var storedAt = timeProvider.GetUtcNow();
var subjectKeys = new HashSet<string>(StringComparer.Ordinal);
var processedCount = 0;
var totalEvents = 0;
long totalHitCount = 0;
// Buffer the content for hashing and parsing
using var buffer = new MemoryStream();
await content.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
// Compute BLAKE3 hash
string batchHash;
if (cryptoHash != null)
{
batchHash = "blake3:" + await cryptoHash.ComputeHashHexAsync(buffer, "BLAKE3-256", cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
}
else
{
// Fallback: generate a deterministic hash based on content length and timestamp
batchHash = $"blake3:{storedAt.ToUnixTimeMilliseconds():x16}{buffer.Length:x16}";
}
// Store to CAS if artifact store is available
StoredRuntimeFactsArtifact? storedArtifact = null;
if (artifactStore != null)
{
var fileName = contentType.Contains("gzip", StringComparison.OrdinalIgnoreCase)
? "runtime-facts.ndjson.gz"
: "runtime-facts.ndjson";
var saveRequest = new RuntimeFactsArtifactSaveRequest(
TenantId: tenantId,
SubjectKey: string.Empty, // Will be populated after parsing
Hash: batchHash.Replace("blake3:", string.Empty),
ContentType: contentType,
FileName: fileName,
BatchSize: buffer.Length,
ProvenanceSource: "runtime-facts-batch");
storedArtifact = await artifactStore.SaveAsync(saveRequest, buffer, cancellationToken).ConfigureAwait(false);
buffer.Position = 0;
}
// Decompress if gzip
Stream parseStream;
if (contentType.Contains("gzip", StringComparison.OrdinalIgnoreCase))
{
var decompressed = new MemoryStream();
await using (var gzip = new GZipStream(buffer, CompressionMode.Decompress, leaveOpen: true))
{
await gzip.CopyToAsync(decompressed, cancellationToken).ConfigureAwait(false);
}
decompressed.Position = 0;
parseStream = decompressed;
}
else
{
parseStream = buffer;
}
// Parse NDJSON and group by subject
var requestsBySubject = new Dictionary<string, RuntimeFactsIngestRequest>(StringComparer.Ordinal);
using var reader = new StreamReader(parseStream, leaveOpen: true);
while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line)
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
try
{
var evt = JsonSerializer.Deserialize<RuntimeFactsBatchEvent>(line, JsonOptions);
if (evt is null || string.IsNullOrWhiteSpace(evt.SymbolId))
{
continue;
}
var subjectKey = evt.Subject?.ToSubjectKey() ?? evt.CallgraphId ?? "unknown";
if (!requestsBySubject.TryGetValue(subjectKey, out var request))
{
request = new RuntimeFactsIngestRequest
{
Subject = evt.Subject ?? new ReachabilitySubject { ScanId = subjectKey },
CallgraphId = evt.CallgraphId ?? subjectKey,
Events = new List<RuntimeFactEvent>(),
Metadata = new Dictionary<string, string?>(StringComparer.Ordinal)
{
["batch.hash"] = batchHash,
["batch.cas_uri"] = storedArtifact?.CasUri,
["tenant_id"] = tenantId
}
};
requestsBySubject[subjectKey] = request;
}
((List<RuntimeFactEvent>)request.Events).Add(new RuntimeFactEvent
{
SymbolId = evt.SymbolId,
CodeId = evt.CodeId,
SymbolDigest = evt.SymbolDigest,
Purl = evt.Purl,
BuildId = evt.BuildId,
LoaderBase = evt.LoaderBase,
ProcessId = evt.ProcessId,
ProcessName = evt.ProcessName,
SocketAddress = evt.SocketAddress,
ContainerId = evt.ContainerId,
EvidenceUri = evt.EvidenceUri,
HitCount = Math.Max(evt.HitCount, 1),
ObservedAt = evt.ObservedAt,
Metadata = evt.Metadata
});
totalEvents++;
totalHitCount += Math.Max(evt.HitCount, 1);
}
catch (JsonException ex)
{
logger.LogWarning(ex, "Failed to parse NDJSON line in batch ingestion.");
}
}
// Process each subject's request
foreach (var (subjectKey, request) in requestsBySubject)
{
try
{
var response = await IngestAsync(request, cancellationToken).ConfigureAwait(false);
// Update the fact document with batch reference
var existing = await factRepository.GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
if (existing != null && storedArtifact != null)
{
existing.RuntimeFactsBatchUri = storedArtifact.CasUri;
existing.RuntimeFactsBatchHash = batchHash;
await factRepository.UpsertAsync(existing, cancellationToken).ConfigureAwait(false);
}
subjectKeys.Add(subjectKey);
processedCount++;
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to ingest batch for subject {SubjectKey}.", subjectKey);
}
}
logger.LogInformation(
"Batch ingestion completed: {ProcessedCount} subjects, {TotalEvents} events, {TotalHitCount} hits (hash={BatchHash}, tenant={TenantId}).",
processedCount,
totalEvents,
totalHitCount,
batchHash,
tenantId);
return new RuntimeFactsBatchIngestResponse
{
CasUri = storedArtifact?.CasUri ?? $"cas://reachability/runtime-facts/{batchHash.Replace("blake3:", string.Empty)}",
BatchHash = batchHash,
ProcessedCount = processedCount,
TotalEvents = totalEvents,
TotalHitCount = totalHitCount,
SubjectKeys = subjectKeys.ToList(),
StoredAt = storedAt
};
}
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
/// <summary>
/// NDJSON batch event structure for runtime facts.
/// </summary>
private sealed class RuntimeFactsBatchEvent
{
public string? SymbolId { get; set; }
public string? CodeId { get; set; }
public string? SymbolDigest { get; set; }
public string? Purl { get; set; }
public string? BuildId { get; set; }
public string? LoaderBase { get; set; }
public int? ProcessId { get; set; }
public string? ProcessName { get; set; }
public string? SocketAddress { get; set; }
public string? ContainerId { get; set; }
public string? EvidenceUri { get; set; }
public int HitCount { get; set; } = 1;
public DateTimeOffset? ObservedAt { get; set; }
public Dictionary<string, string?>? Metadata { get; set; }
public ReachabilitySubject? Subject { get; set; }
public string? CallgraphId { get; set; }
}
private static void ValidateRequest(RuntimeFactsIngestRequest request)
{
if (request.Subject is null)

View File

@@ -0,0 +1,160 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Signals.Options;
using StellaOps.Signals.Storage.Models;
namespace StellaOps.Signals.Storage;
/// <summary>
/// Stores runtime-facts batch artifacts on the local filesystem.
/// CAS paths: cas://reachability/runtime-facts/{hash}
/// </summary>
internal sealed class FileSystemRuntimeFactsArtifactStore : IRuntimeFactsArtifactStore
{
private const string DefaultFileName = "runtime-facts.ndjson";
private readonly SignalsArtifactStorageOptions _storageOptions;
private readonly ILogger<FileSystemRuntimeFactsArtifactStore> _logger;
public FileSystemRuntimeFactsArtifactStore(
IOptions<SignalsOptions> options,
ILogger<FileSystemRuntimeFactsArtifactStore> logger)
{
ArgumentNullException.ThrowIfNull(options);
_storageOptions = options.Value.Storage;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<StoredRuntimeFactsArtifact> SaveAsync(
RuntimeFactsArtifactSaveRequest request,
Stream content,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
ArgumentNullException.ThrowIfNull(content);
var hash = NormalizeHash(request.Hash);
if (string.IsNullOrWhiteSpace(hash))
{
throw new InvalidOperationException("Runtime-facts artifact hash is required for CAS storage.");
}
var casDirectory = GetCasDirectory(hash);
Directory.CreateDirectory(casDirectory);
var fileName = SanitizeFileName(string.IsNullOrWhiteSpace(request.FileName) ? DefaultFileName : request.FileName);
var destinationPath = Path.Combine(casDirectory, fileName);
await using (var fileStream = File.Create(destinationPath))
{
await content.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false);
}
var fileInfo = new FileInfo(destinationPath);
var casUri = $"cas://reachability/runtime-facts/{hash}";
_logger.LogInformation(
"Stored runtime-facts artifact at {Path} (length={Length}, hash={Hash}, tenant={TenantId}).",
destinationPath,
fileInfo.Length,
hash,
request.TenantId);
return new StoredRuntimeFactsArtifact(
Path.GetRelativePath(_storageOptions.RootPath, destinationPath),
fileInfo.Length,
hash,
request.ContentType,
casUri);
}
public Task<Stream?> GetAsync(string hash, CancellationToken cancellationToken = default)
{
var normalizedHash = NormalizeHash(hash);
if (string.IsNullOrWhiteSpace(normalizedHash))
{
return Task.FromResult<Stream?>(null);
}
var casDirectory = GetCasDirectory(normalizedHash);
var filePath = Path.Combine(casDirectory, DefaultFileName);
// Also check for gzip variant
if (!File.Exists(filePath))
{
filePath = Path.Combine(casDirectory, "runtime-facts.ndjson.gz");
}
if (!File.Exists(filePath))
{
_logger.LogDebug("Runtime-facts artifact {Hash} not found at {Path}.", normalizedHash, filePath);
return Task.FromResult<Stream?>(null);
}
var content = new MemoryStream();
using (var fileStream = File.OpenRead(filePath))
{
fileStream.CopyTo(content);
}
content.Position = 0;
_logger.LogDebug("Retrieved runtime-facts artifact {Hash} from {Path}.", normalizedHash, filePath);
return Task.FromResult<Stream?>(content);
}
public Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default)
{
var normalizedHash = NormalizeHash(hash);
if (string.IsNullOrWhiteSpace(normalizedHash))
{
return Task.FromResult(false);
}
var casDirectory = GetCasDirectory(normalizedHash);
var defaultPath = Path.Combine(casDirectory, DefaultFileName);
var gzipPath = Path.Combine(casDirectory, "runtime-facts.ndjson.gz");
var exists = File.Exists(defaultPath) || File.Exists(gzipPath);
_logger.LogDebug("Runtime-facts artifact {Hash} exists={Exists}.", normalizedHash, exists);
return Task.FromResult(exists);
}
public Task<bool> DeleteAsync(string hash, CancellationToken cancellationToken = default)
{
var normalizedHash = NormalizeHash(hash);
if (string.IsNullOrWhiteSpace(normalizedHash))
{
return Task.FromResult(false);
}
var casDirectory = GetCasDirectory(normalizedHash);
if (!Directory.Exists(casDirectory))
{
return Task.FromResult(false);
}
try
{
Directory.Delete(casDirectory, recursive: true);
_logger.LogInformation("Deleted runtime-facts artifact {Hash}.", normalizedHash);
return Task.FromResult(true);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to delete runtime-facts artifact {Hash}.", normalizedHash);
return Task.FromResult(false);
}
}
private string GetCasDirectory(string hash)
{
var prefix = hash.Length >= 2 ? hash[..2] : hash;
return Path.Combine(_storageOptions.RootPath, "cas", "reachability", "runtime-facts", prefix, hash);
}
private static string? NormalizeHash(string? hash)
=> hash?.Trim().ToLowerInvariant();
private static string SanitizeFileName(string value)
=> string.Join('_', value.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)).ToLowerInvariant();
}

View File

@@ -0,0 +1,46 @@
using StellaOps.Signals.Storage.Models;
namespace StellaOps.Signals.Storage;
/// <summary>
/// Persists and retrieves runtime-facts batch artifacts from content-addressable storage.
/// CAS paths follow: cas://reachability/runtime-facts/{hash}
/// </summary>
public interface IRuntimeFactsArtifactStore
{
/// <summary>
/// Stores a runtime-facts batch artifact.
/// </summary>
/// <param name="request">Metadata about the artifact to store.</param>
/// <param name="content">The artifact content stream (NDJSON or gzip compressed).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Information about the stored artifact including CAS URI.</returns>
Task<StoredRuntimeFactsArtifact> SaveAsync(
RuntimeFactsArtifactSaveRequest request,
Stream content,
CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves a runtime-facts artifact by its BLAKE3 hash.
/// </summary>
/// <param name="hash">The BLAKE3 hash of the artifact (blake3:{hex}).</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The artifact content stream, or null if not found.</returns>
Task<Stream?> GetAsync(string hash, CancellationToken cancellationToken = default);
/// <summary>
/// Checks if a runtime-facts artifact exists.
/// </summary>
/// <param name="hash">The BLAKE3 hash of the artifact.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>True if the artifact exists.</returns>
Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default);
/// <summary>
/// Deletes a runtime-facts artifact if it exists.
/// </summary>
/// <param name="hash">The BLAKE3 hash of the artifact.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>True if the artifact was deleted, false if it did not exist.</returns>
Task<bool> DeleteAsync(string hash, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,13 @@
namespace StellaOps.Signals.Storage.Models;
/// <summary>
/// Context required to persist a runtime-facts artifact batch.
/// </summary>
public sealed record RuntimeFactsArtifactSaveRequest(
string TenantId,
string SubjectKey,
string Hash,
string ContentType,
string FileName,
long BatchSize,
string? ProvenanceSource);

View File

@@ -0,0 +1,11 @@
namespace StellaOps.Signals.Storage.Models;
/// <summary>
/// Result returned after storing a runtime-facts artifact.
/// </summary>
public sealed record StoredRuntimeFactsArtifact(
string Path,
long Length,
string Hash,
string ContentType,
string CasUri);