feat: Implement Wine CSP HTTP provider for GOST cryptographic operations
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
- Added WineCspHttpProvider class to interface with Wine-hosted CryptoPro CSP. - Implemented ICryptoProvider, ICryptoProviderDiagnostics, and IDisposable interfaces. - Introduced WineCspHttpSigner and WineCspHttpHasher for signing and hashing operations. - Created WineCspProviderOptions for configuration settings including service URL and key options. - Developed CryptoProGostSigningService to handle GOST signing operations and key management. - Implemented HTTP service for the Wine CSP with endpoints for signing, verification, and hashing. - Added Swagger documentation for API endpoints. - Included health checks and error handling for service availability. - Established DTOs for request and response models in the service.
This commit is contained in:
343
src/Signals/StellaOps.Signals/Models/AocProvenance.cs
Normal file
343
src/Signals/StellaOps.Signals/Models/AocProvenance.cs
Normal file
@@ -0,0 +1,343 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
/// <summary>
|
||||
/// AOC (Aggregation-Only Contract) provenance feed for runtime facts ingestion (SGSI0101).
|
||||
/// Conforms to docs/schemas/provenance-feed.schema.json.
|
||||
/// </summary>
|
||||
public sealed class ProvenanceFeed
|
||||
{
|
||||
public const int CurrentSchemaVersion = 1;
|
||||
|
||||
[BsonElement("schemaVersion")]
|
||||
[JsonPropertyName("schemaVersion")]
|
||||
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
|
||||
|
||||
[BsonElement("feedId")]
|
||||
[JsonPropertyName("feedId")]
|
||||
public string FeedId { get; init; } = Guid.NewGuid().ToString("D");
|
||||
|
||||
[BsonElement("feedType")]
|
||||
[JsonPropertyName("feedType")]
|
||||
public ProvenanceFeedType FeedType { get; init; } = ProvenanceFeedType.RuntimeFacts;
|
||||
|
||||
[BsonElement("generatedAt")]
|
||||
[JsonPropertyName("generatedAt")]
|
||||
public DateTimeOffset GeneratedAt { get; init; } = DateTimeOffset.UtcNow;
|
||||
|
||||
[BsonElement("sourceService")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("sourceService")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? SourceService { get; init; }
|
||||
|
||||
[BsonElement("tenantId")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("tenantId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? TenantId { get; init; }
|
||||
|
||||
[BsonElement("correlationId")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("correlationId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? CorrelationId { get; init; }
|
||||
|
||||
[BsonElement("records")]
|
||||
[JsonPropertyName("records")]
|
||||
public List<ProvenanceRecord> Records { get; init; } = new();
|
||||
|
||||
[BsonElement("metadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("metadata")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public Dictionary<string, string?>? Metadata { get; init; }
|
||||
|
||||
[BsonElement("attestation")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("attestation")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public FeedAttestation? Attestation { get; init; }
|
||||
}
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum ProvenanceFeedType
|
||||
{
|
||||
[JsonPropertyName("RUNTIME_FACTS")]
|
||||
RuntimeFacts,
|
||||
|
||||
[JsonPropertyName("SIGNAL_ENRICHMENT")]
|
||||
SignalEnrichment,
|
||||
|
||||
[JsonPropertyName("CAS_PROMOTION")]
|
||||
CasPromotion,
|
||||
|
||||
[JsonPropertyName("SCORING_OUTPUT")]
|
||||
ScoringOutput,
|
||||
|
||||
[JsonPropertyName("AUTHORITY_SCOPES")]
|
||||
AuthorityScopes
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Individual provenance record within a feed.
|
||||
/// </summary>
|
||||
public sealed class ProvenanceRecord
|
||||
{
|
||||
[BsonElement("recordId")]
|
||||
[JsonPropertyName("recordId")]
|
||||
public string RecordId { get; init; } = Guid.NewGuid().ToString("D");
|
||||
|
||||
[BsonElement("recordType")]
|
||||
[JsonPropertyName("recordType")]
|
||||
public string RecordType { get; init; } = string.Empty;
|
||||
|
||||
[BsonElement("subject")]
|
||||
[JsonPropertyName("subject")]
|
||||
public ProvenanceSubject Subject { get; init; } = new();
|
||||
|
||||
[BsonElement("occurredAt")]
|
||||
[JsonPropertyName("occurredAt")]
|
||||
public DateTimeOffset OccurredAt { get; init; }
|
||||
|
||||
[BsonElement("observedBy")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("observedBy")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? ObservedBy { get; init; }
|
||||
|
||||
[BsonElement("confidence")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("confidence")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public double? Confidence { get; init; }
|
||||
|
||||
[BsonElement("facts")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("facts")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public RuntimeProvenanceFacts? Facts { get; init; }
|
||||
|
||||
[BsonElement("evidence")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("evidence")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public RecordEvidence? Evidence { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Subject of a provenance record.
|
||||
/// </summary>
|
||||
public sealed class ProvenanceSubject
|
||||
{
|
||||
[BsonElement("type")]
|
||||
[JsonPropertyName("type")]
|
||||
public ProvenanceSubjectType Type { get; init; } = ProvenanceSubjectType.Package;
|
||||
|
||||
[BsonElement("identifier")]
|
||||
[JsonPropertyName("identifier")]
|
||||
public string Identifier { get; init; } = string.Empty;
|
||||
|
||||
[BsonElement("digest")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("digest")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Digest { get; init; }
|
||||
|
||||
[BsonElement("namespace")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("namespace")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Namespace { get; init; }
|
||||
}
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum ProvenanceSubjectType
|
||||
{
|
||||
[JsonPropertyName("CONTAINER")]
|
||||
Container,
|
||||
|
||||
[JsonPropertyName("PROCESS")]
|
||||
Process,
|
||||
|
||||
[JsonPropertyName("PACKAGE")]
|
||||
Package,
|
||||
|
||||
[JsonPropertyName("FILE")]
|
||||
File,
|
||||
|
||||
[JsonPropertyName("NETWORK")]
|
||||
Network,
|
||||
|
||||
[JsonPropertyName("IMAGE")]
|
||||
Image
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runtime-specific provenance facts.
|
||||
/// </summary>
|
||||
public sealed class RuntimeProvenanceFacts
|
||||
{
|
||||
[BsonElement("symbolId")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("symbolId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? SymbolId { get; init; }
|
||||
|
||||
[BsonElement("processName")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("processName")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? ProcessName { get; init; }
|
||||
|
||||
[BsonElement("processId")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("processId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? ProcessId { get; init; }
|
||||
|
||||
[BsonElement("socketAddress")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("socketAddress")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? SocketAddress { get; init; }
|
||||
|
||||
[BsonElement("containerId")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("containerId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? ContainerId { get; init; }
|
||||
|
||||
[BsonElement("hitCount")]
|
||||
[JsonPropertyName("hitCount")]
|
||||
public int HitCount { get; init; }
|
||||
|
||||
[BsonElement("purl")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("purl")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Purl { get; init; }
|
||||
|
||||
[BsonElement("codeId")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("codeId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? CodeId { get; init; }
|
||||
|
||||
[BsonElement("buildId")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("buildId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? BuildId { get; init; }
|
||||
|
||||
[BsonElement("loaderBase")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("loaderBase")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? LoaderBase { get; init; }
|
||||
|
||||
[BsonElement("metadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("metadata")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public Dictionary<string, string?>? Metadata { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evidence supporting a provenance record.
|
||||
/// </summary>
|
||||
public sealed class RecordEvidence
|
||||
{
|
||||
[BsonElement("sourceDigest")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("sourceDigest")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? SourceDigest { get; init; }
|
||||
|
||||
[BsonElement("captureMethod")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("captureMethod")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public EvidenceCaptureMethod? CaptureMethod { get; init; }
|
||||
|
||||
[BsonElement("rawDataRef")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("rawDataRef")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? RawDataRef { get; init; }
|
||||
}
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum EvidenceCaptureMethod
|
||||
{
|
||||
[JsonPropertyName("eBPF")]
|
||||
EBpf,
|
||||
|
||||
[JsonPropertyName("PROC_SCAN")]
|
||||
ProcScan,
|
||||
|
||||
[JsonPropertyName("API_CALL")]
|
||||
ApiCall,
|
||||
|
||||
[JsonPropertyName("LOG_ANALYSIS")]
|
||||
LogAnalysis,
|
||||
|
||||
[JsonPropertyName("STATIC_ANALYSIS")]
|
||||
StaticAnalysis
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attestation metadata for a provenance feed.
|
||||
/// </summary>
|
||||
public sealed class FeedAttestation
|
||||
{
|
||||
[BsonElement("predicateType")]
|
||||
[JsonPropertyName("predicateType")]
|
||||
public string PredicateType { get; init; } = "https://stella.ops/attestation/provenance-feed/v1";
|
||||
|
||||
[BsonElement("signedAt")]
|
||||
[JsonPropertyName("signedAt")]
|
||||
public DateTimeOffset SignedAt { get; init; }
|
||||
|
||||
[BsonElement("keyId")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("keyId")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? KeyId { get; init; }
|
||||
|
||||
[BsonElement("envelopeDigest")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("envelopeDigest")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? EnvelopeDigest { get; init; }
|
||||
|
||||
[BsonElement("transparencyLog")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("transparencyLog")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? TransparencyLog { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Context facts container stored on ReachabilityFactDocument.
|
||||
/// </summary>
|
||||
public sealed class ContextFacts
|
||||
{
|
||||
[BsonElement("provenance")]
|
||||
[BsonIgnoreIfNull]
|
||||
[JsonPropertyName("provenance")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public ProvenanceFeed? Provenance { get; set; }
|
||||
|
||||
[BsonElement("lastUpdatedAt")]
|
||||
[JsonPropertyName("lastUpdatedAt")]
|
||||
public DateTimeOffset LastUpdatedAt { get; set; }
|
||||
|
||||
[BsonElement("recordCount")]
|
||||
[JsonPropertyName("recordCount")]
|
||||
public int RecordCount { get; set; }
|
||||
}
|
||||
@@ -31,6 +31,10 @@ public sealed class ReachabilityFactDocument
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
|
||||
[BsonElement("contextFacts")]
|
||||
[BsonIgnoreIfNull]
|
||||
public ContextFacts? ContextFacts { get; set; }
|
||||
|
||||
[BsonElement("score")]
|
||||
public double Score { get; set; }
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Signals.Options;
|
||||
@@ -9,18 +10,144 @@ namespace StellaOps.Signals.Options;
|
||||
public sealed class SignalsArtifactStorageOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Root directory used to persist raw callgraph artifacts.
|
||||
/// Storage driver: "filesystem" (default) or "rustfs".
|
||||
/// </summary>
|
||||
public string Driver { get; set; } = SignalsStorageDrivers.FileSystem;
|
||||
|
||||
/// <summary>
|
||||
/// Root directory used to persist raw callgraph artifacts (filesystem driver).
|
||||
/// </summary>
|
||||
public string RootPath { get; set; } = Path.Combine(AppContext.BaseDirectory, "callgraph-artifacts");
|
||||
|
||||
/// <summary>
|
||||
/// Bucket name for CAS storage (RustFS driver).
|
||||
/// Per CAS contract, signals uses "signals-data" bucket.
|
||||
/// </summary>
|
||||
public string BucketName { get; set; } = "signals-data";
|
||||
|
||||
/// <summary>
|
||||
/// Root prefix within the bucket for callgraph artifacts.
|
||||
/// </summary>
|
||||
public string RootPrefix { get; set; } = "callgraphs";
|
||||
|
||||
/// <summary>
|
||||
/// RustFS-specific options.
|
||||
/// </summary>
|
||||
public SignalsRustFsOptions RustFs { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Additional headers to include in storage requests.
|
||||
/// </summary>
|
||||
public IDictionary<string, string> Headers { get; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the filesystem driver is configured.
|
||||
/// </summary>
|
||||
public bool IsFileSystemDriver()
|
||||
=> string.Equals(Driver, SignalsStorageDrivers.FileSystem, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if the RustFS driver is configured.
|
||||
/// </summary>
|
||||
public bool IsRustFsDriver()
|
||||
=> string.Equals(Driver, SignalsStorageDrivers.RustFs, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Validates the configured values.
|
||||
/// </summary>
|
||||
public void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(RootPath))
|
||||
if (!IsFileSystemDriver() && !IsRustFsDriver())
|
||||
{
|
||||
throw new InvalidOperationException("Signals artifact storage path must be configured.");
|
||||
throw new InvalidOperationException($"Signals storage driver '{Driver}' is not supported. Use '{SignalsStorageDrivers.FileSystem}' or '{SignalsStorageDrivers.RustFs}'.");
|
||||
}
|
||||
|
||||
if (IsFileSystemDriver() && string.IsNullOrWhiteSpace(RootPath))
|
||||
{
|
||||
throw new InvalidOperationException("Signals artifact storage path must be configured for filesystem driver.");
|
||||
}
|
||||
|
||||
if (IsRustFsDriver())
|
||||
{
|
||||
RustFs ??= new SignalsRustFsOptions();
|
||||
RustFs.Validate();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(BucketName))
|
||||
{
|
||||
throw new InvalidOperationException("Signals storage bucket name must be configured for RustFS driver.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// RustFS-specific configuration options.
|
||||
/// </summary>
|
||||
public sealed class SignalsRustFsOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Base URL for the RustFS service (e.g., http://localhost:8180/api/v1).
|
||||
/// </summary>
|
||||
public string BaseUrl { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Allow insecure TLS connections (development only).
|
||||
/// </summary>
|
||||
public bool AllowInsecureTls { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// API key for authentication.
|
||||
/// </summary>
|
||||
public string? ApiKey { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Header name for the API key (e.g., "X-API-Key").
|
||||
/// </summary>
|
||||
public string ApiKeyHeader { get; set; } = "X-API-Key";
|
||||
|
||||
/// <summary>
|
||||
/// HTTP request timeout.
|
||||
/// </summary>
|
||||
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// Validates the configured values.
|
||||
/// </summary>
|
||||
public void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(BaseUrl))
|
||||
{
|
||||
throw new InvalidOperationException("RustFS baseUrl must be configured.");
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(BaseUrl, UriKind.Absolute, out var uri))
|
||||
{
|
||||
throw new InvalidOperationException("RustFS baseUrl must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase)
|
||||
&& !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("RustFS baseUrl must use HTTP or HTTPS.");
|
||||
}
|
||||
|
||||
if (Timeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException("RustFS timeout must be greater than zero.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ApiKeyHeader) && string.IsNullOrWhiteSpace(ApiKey))
|
||||
{
|
||||
throw new InvalidOperationException("RustFS API key header name requires a non-empty API key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Supported storage driver names.
|
||||
/// </summary>
|
||||
public static class SignalsStorageDrivers
|
||||
{
|
||||
public const string FileSystem = "filesystem";
|
||||
public const string RustFs = "rustfs";
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,6 +17,7 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
private readonly IReachabilityCache cache;
|
||||
private readonly IEventsPublisher eventsPublisher;
|
||||
private readonly IReachabilityScoringService scoringService;
|
||||
private readonly IRuntimeFactsProvenanceNormalizer provenanceNormalizer;
|
||||
private readonly ILogger<RuntimeFactsIngestionService> logger;
|
||||
|
||||
public RuntimeFactsIngestionService(
|
||||
@@ -25,6 +26,7 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
IReachabilityCache cache,
|
||||
IEventsPublisher eventsPublisher,
|
||||
IReachabilityScoringService scoringService,
|
||||
IRuntimeFactsProvenanceNormalizer provenanceNormalizer,
|
||||
ILogger<RuntimeFactsIngestionService> logger)
|
||||
{
|
||||
this.factRepository = factRepository ?? throw new ArgumentNullException(nameof(factRepository));
|
||||
@@ -32,6 +34,7 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
this.cache = cache ?? throw new ArgumentNullException(nameof(cache));
|
||||
this.eventsPublisher = eventsPublisher ?? throw new ArgumentNullException(nameof(eventsPublisher));
|
||||
this.scoringService = scoringService ?? throw new ArgumentNullException(nameof(scoringService));
|
||||
this.provenanceNormalizer = provenanceNormalizer ?? throw new ArgumentNullException(nameof(provenanceNormalizer));
|
||||
this.logger = logger ?? NullLogger<RuntimeFactsIngestionService>.Instance;
|
||||
}
|
||||
|
||||
@@ -62,6 +65,14 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
document.Metadata["provenance.ingestedAt"] = document.ComputedAt.ToString("O");
|
||||
document.Metadata["provenance.callgraphId"] = request.CallgraphId;
|
||||
|
||||
// Populate context_facts with AOC provenance (SIGNALS-24-003)
|
||||
document.ContextFacts = provenanceNormalizer.CreateContextFacts(
|
||||
request.Events,
|
||||
request.Subject,
|
||||
request.CallgraphId,
|
||||
request.Metadata,
|
||||
document.ComputedAt);
|
||||
|
||||
var persisted = await factRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
await cache.SetAsync(persisted, cancellationToken).ConfigureAwait(false);
|
||||
await eventsPublisher.PublishFactUpdatedAsync(persisted, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Signals.Models;
|
||||
|
||||
namespace StellaOps.Signals.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes runtime fact events into AOC provenance records per SIGNALS-24-003.
|
||||
/// Converts process, socket, and container metadata to <see cref="ProvenanceRecord"/> format.
|
||||
/// </summary>
|
||||
public interface IRuntimeFactsProvenanceNormalizer
|
||||
{
|
||||
/// <summary>
|
||||
/// Normalizes runtime fact events into a provenance feed.
|
||||
/// </summary>
|
||||
ProvenanceFeed NormalizeToFeed(
|
||||
IEnumerable<RuntimeFactEvent> events,
|
||||
ReachabilitySubject subject,
|
||||
string callgraphId,
|
||||
Dictionary<string, string?>? metadata,
|
||||
DateTimeOffset generatedAt);
|
||||
|
||||
/// <summary>
|
||||
/// Creates or updates context facts from runtime events.
|
||||
/// </summary>
|
||||
ContextFacts CreateContextFacts(
|
||||
IEnumerable<RuntimeFactEvent> events,
|
||||
ReachabilitySubject subject,
|
||||
string callgraphId,
|
||||
Dictionary<string, string?>? metadata,
|
||||
DateTimeOffset timestamp);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of runtime facts provenance normalizer.
|
||||
/// </summary>
|
||||
public sealed class RuntimeFactsProvenanceNormalizer : IRuntimeFactsProvenanceNormalizer
|
||||
{
|
||||
private const string SourceService = "signals-runtime-ingestion";
|
||||
private const double DefaultConfidence = 0.95;
|
||||
|
||||
public ProvenanceFeed NormalizeToFeed(
|
||||
IEnumerable<RuntimeFactEvent> events,
|
||||
ReachabilitySubject subject,
|
||||
string callgraphId,
|
||||
Dictionary<string, string?>? metadata,
|
||||
DateTimeOffset generatedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(events);
|
||||
ArgumentNullException.ThrowIfNull(subject);
|
||||
|
||||
var eventsList = events.Where(e => e is not null && !string.IsNullOrWhiteSpace(e.SymbolId)).ToList();
|
||||
var records = new List<ProvenanceRecord>(eventsList.Count);
|
||||
|
||||
foreach (var evt in eventsList)
|
||||
{
|
||||
var record = NormalizeEvent(evt, subject, callgraphId, generatedAt);
|
||||
if (record is not null)
|
||||
{
|
||||
records.Add(record);
|
||||
}
|
||||
}
|
||||
|
||||
var feedMetadata = new Dictionary<string, string?>(StringComparer.Ordinal)
|
||||
{
|
||||
["aoc.version"] = "1",
|
||||
["aoc.contract"] = "SGSI0101",
|
||||
["callgraphId"] = callgraphId,
|
||||
["subjectKey"] = subject.ToSubjectKey()
|
||||
};
|
||||
|
||||
if (metadata is not null)
|
||||
{
|
||||
foreach (var (key, value) in metadata)
|
||||
{
|
||||
feedMetadata[$"request.{key}"] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return new ProvenanceFeed
|
||||
{
|
||||
SchemaVersion = ProvenanceFeed.CurrentSchemaVersion,
|
||||
FeedId = Guid.NewGuid().ToString("D"),
|
||||
FeedType = ProvenanceFeedType.RuntimeFacts,
|
||||
GeneratedAt = generatedAt,
|
||||
SourceService = SourceService,
|
||||
CorrelationId = callgraphId,
|
||||
Records = records,
|
||||
Metadata = feedMetadata
|
||||
};
|
||||
}
|
||||
|
||||
public ContextFacts CreateContextFacts(
|
||||
IEnumerable<RuntimeFactEvent> events,
|
||||
ReachabilitySubject subject,
|
||||
string callgraphId,
|
||||
Dictionary<string, string?>? metadata,
|
||||
DateTimeOffset timestamp)
|
||||
{
|
||||
var feed = NormalizeToFeed(events, subject, callgraphId, metadata, timestamp);
|
||||
|
||||
return new ContextFacts
|
||||
{
|
||||
Provenance = feed,
|
||||
LastUpdatedAt = timestamp,
|
||||
RecordCount = feed.Records.Count
|
||||
};
|
||||
}
|
||||
|
||||
private static ProvenanceRecord? NormalizeEvent(
|
||||
RuntimeFactEvent evt,
|
||||
ReachabilitySubject subject,
|
||||
string callgraphId,
|
||||
DateTimeOffset generatedAt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(evt.SymbolId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var recordType = DetermineRecordType(evt);
|
||||
var subjectType = DetermineSubjectType(evt, subject);
|
||||
|
||||
var provenanceSubject = new ProvenanceSubject
|
||||
{
|
||||
Type = subjectType,
|
||||
Identifier = BuildSubjectIdentifier(evt, subject),
|
||||
Digest = NormalizeDigest(evt.SymbolDigest),
|
||||
Namespace = ExtractNamespace(evt.ContainerId, subject)
|
||||
};
|
||||
|
||||
var facts = new RuntimeProvenanceFacts
|
||||
{
|
||||
SymbolId = evt.SymbolId.Trim(),
|
||||
ProcessName = Normalize(evt.ProcessName),
|
||||
ProcessId = evt.ProcessId,
|
||||
SocketAddress = Normalize(evt.SocketAddress),
|
||||
ContainerId = Normalize(evt.ContainerId),
|
||||
HitCount = Math.Max(evt.HitCount, 1),
|
||||
Purl = Normalize(evt.Purl),
|
||||
CodeId = Normalize(evt.CodeId),
|
||||
BuildId = Normalize(evt.BuildId),
|
||||
LoaderBase = Normalize(evt.LoaderBase),
|
||||
Metadata = evt.Metadata
|
||||
};
|
||||
|
||||
var evidence = BuildEvidence(evt);
|
||||
|
||||
return new ProvenanceRecord
|
||||
{
|
||||
RecordId = Guid.NewGuid().ToString("D"),
|
||||
RecordType = recordType,
|
||||
Subject = provenanceSubject,
|
||||
OccurredAt = evt.ObservedAt ?? generatedAt,
|
||||
ObservedBy = DetermineObserver(evt),
|
||||
Confidence = ComputeConfidence(evt),
|
||||
Facts = facts,
|
||||
Evidence = evidence
|
||||
};
|
||||
}
|
||||
|
||||
private static string DetermineRecordType(RuntimeFactEvent evt)
|
||||
{
|
||||
// Determine record type based on available metadata
|
||||
if (!string.IsNullOrWhiteSpace(evt.ProcessName) || evt.ProcessId.HasValue)
|
||||
{
|
||||
return "runtime.process.observed";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(evt.SocketAddress))
|
||||
{
|
||||
return "runtime.network.connection";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(evt.ContainerId))
|
||||
{
|
||||
return "runtime.container.activity";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(evt.Purl))
|
||||
{
|
||||
return "runtime.package.loaded";
|
||||
}
|
||||
|
||||
return "runtime.symbol.invoked";
|
||||
}
|
||||
|
||||
private static ProvenanceSubjectType DetermineSubjectType(RuntimeFactEvent evt, ReachabilitySubject subject)
|
||||
{
|
||||
// Priority: container > process > package > file
|
||||
if (!string.IsNullOrWhiteSpace(evt.ContainerId))
|
||||
{
|
||||
return ProvenanceSubjectType.Container;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(evt.ProcessName) || evt.ProcessId.HasValue)
|
||||
{
|
||||
return ProvenanceSubjectType.Process;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(evt.Purl))
|
||||
{
|
||||
return ProvenanceSubjectType.Package;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(subject.ImageDigest))
|
||||
{
|
||||
return ProvenanceSubjectType.Image;
|
||||
}
|
||||
|
||||
return ProvenanceSubjectType.Package;
|
||||
}
|
||||
|
||||
private static string BuildSubjectIdentifier(RuntimeFactEvent evt, ReachabilitySubject subject)
|
||||
{
|
||||
// Build identifier based on available data
|
||||
if (!string.IsNullOrWhiteSpace(evt.Purl))
|
||||
{
|
||||
return evt.Purl.Trim();
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(evt.ContainerId))
|
||||
{
|
||||
return evt.ContainerId.Trim();
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(subject.ImageDigest))
|
||||
{
|
||||
return subject.ImageDigest;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(subject.Component))
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(subject.Version)
|
||||
? subject.Component
|
||||
: $"{subject.Component}@{subject.Version}";
|
||||
}
|
||||
|
||||
return evt.SymbolId.Trim();
|
||||
}
|
||||
|
||||
private static string? NormalizeDigest(string? digest)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(digest))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = digest.Trim();
|
||||
|
||||
// Ensure sha256: prefix for valid hex digests
|
||||
if (trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return trimmed.ToLowerInvariant();
|
||||
}
|
||||
|
||||
// If it looks like a hex digest (64 chars), add prefix
|
||||
if (trimmed.Length == 64 && IsHexString(trimmed))
|
||||
{
|
||||
return $"sha256:{trimmed.ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private static bool IsHexString(string value)
|
||||
{
|
||||
foreach (var c in value)
|
||||
{
|
||||
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string? ExtractNamespace(string? containerId, ReachabilitySubject subject)
|
||||
{
|
||||
// Try to extract namespace from container ID or subject metadata
|
||||
if (!string.IsNullOrWhiteSpace(containerId) && containerId.Contains('/'))
|
||||
{
|
||||
var parts = containerId.Split('/');
|
||||
if (parts.Length > 1)
|
||||
{
|
||||
return parts[0];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static RecordEvidence? BuildEvidence(RuntimeFactEvent evt)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(evt.EvidenceUri) && string.IsNullOrWhiteSpace(evt.SymbolDigest))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var captureMethod = DetermineCaptureMethod(evt);
|
||||
|
||||
return new RecordEvidence
|
||||
{
|
||||
SourceDigest = NormalizeDigest(evt.SymbolDigest),
|
||||
CaptureMethod = captureMethod,
|
||||
RawDataRef = Normalize(evt.EvidenceUri)
|
||||
};
|
||||
}
|
||||
|
||||
private static EvidenceCaptureMethod? DetermineCaptureMethod(RuntimeFactEvent evt)
|
||||
{
|
||||
// Infer capture method from event metadata
|
||||
if (evt.Metadata is not null)
|
||||
{
|
||||
if (evt.Metadata.TryGetValue("captureMethod", out var method) && !string.IsNullOrWhiteSpace(method))
|
||||
{
|
||||
return method.ToUpperInvariant() switch
|
||||
{
|
||||
"EBPF" => EvidenceCaptureMethod.EBpf,
|
||||
"PROC_SCAN" => EvidenceCaptureMethod.ProcScan,
|
||||
"API_CALL" => EvidenceCaptureMethod.ApiCall,
|
||||
"LOG_ANALYSIS" => EvidenceCaptureMethod.LogAnalysis,
|
||||
"STATIC_ANALYSIS" => EvidenceCaptureMethod.StaticAnalysis,
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Default based on available data
|
||||
if (evt.ProcessId.HasValue || !string.IsNullOrWhiteSpace(evt.ProcessName))
|
||||
{
|
||||
return EvidenceCaptureMethod.ProcScan;
|
||||
}
|
||||
|
||||
return EvidenceCaptureMethod.ApiCall;
|
||||
}
|
||||
|
||||
private static string? DetermineObserver(RuntimeFactEvent evt)
|
||||
{
|
||||
if (evt.Metadata is not null && evt.Metadata.TryGetValue("observer", out var observer))
|
||||
{
|
||||
return Normalize(observer);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(evt.ContainerId))
|
||||
{
|
||||
return "container-runtime-agent";
|
||||
}
|
||||
|
||||
if (evt.ProcessId.HasValue)
|
||||
{
|
||||
return "process-monitor-agent";
|
||||
}
|
||||
|
||||
return "signals-ingestion";
|
||||
}
|
||||
|
||||
private static double ComputeConfidence(RuntimeFactEvent evt)
|
||||
{
|
||||
// Base confidence
|
||||
var confidence = DefaultConfidence;
|
||||
|
||||
// Adjust based on available evidence
|
||||
if (!string.IsNullOrWhiteSpace(evt.SymbolDigest))
|
||||
{
|
||||
confidence = Math.Min(confidence + 0.02, 1.0);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(evt.EvidenceUri))
|
||||
{
|
||||
confidence = Math.Min(confidence + 0.01, 1.0);
|
||||
}
|
||||
|
||||
if (evt.ProcessId.HasValue && !string.IsNullOrWhiteSpace(evt.ProcessName))
|
||||
{
|
||||
confidence = Math.Min(confidence + 0.01, 1.0);
|
||||
}
|
||||
|
||||
return Math.Round(confidence, 2);
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value) =>
|
||||
string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
}
|
||||
@@ -1,46 +1,48 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Stores callgraph artifacts on the local filesystem.
|
||||
/// </summary>
|
||||
internal sealed class FileSystemCallgraphArtifactStore : ICallgraphArtifactStore
|
||||
{
|
||||
private readonly SignalsArtifactStorageOptions storageOptions;
|
||||
private readonly ILogger<FileSystemCallgraphArtifactStore> logger;
|
||||
|
||||
public FileSystemCallgraphArtifactStore(IOptions<SignalsOptions> options, ILogger<FileSystemCallgraphArtifactStore> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
storageOptions = options.Value.Storage;
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Stores callgraph artifacts on the local filesystem.
|
||||
/// </summary>
|
||||
internal sealed class FileSystemCallgraphArtifactStore : ICallgraphArtifactStore
|
||||
{
|
||||
private const string DefaultFileName = "callgraph.json";
|
||||
private const string ManifestFileName = "manifest.json";
|
||||
|
||||
private readonly SignalsArtifactStorageOptions _storageOptions;
|
||||
private readonly ILogger<FileSystemCallgraphArtifactStore> _logger;
|
||||
|
||||
public FileSystemCallgraphArtifactStore(IOptions<SignalsOptions> options, ILogger<FileSystemCallgraphArtifactStore> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_storageOptions = options.Value.Storage;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<StoredCallgraphArtifact> SaveAsync(CallgraphArtifactSaveRequest request, Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var root = storageOptions.RootPath;
|
||||
var hash = request.Hash?.Trim().ToLowerInvariant();
|
||||
var root = _storageOptions.RootPath;
|
||||
var hash = NormalizeHash(request.Hash);
|
||||
if (string.IsNullOrWhiteSpace(hash))
|
||||
{
|
||||
throw new InvalidOperationException("Callgraph artifact hash is required for CAS storage.");
|
||||
}
|
||||
|
||||
var casDirectory = Path.Combine(root, "cas", "reachability", "graphs", hash.Substring(0, Math.Min(hash.Length, 2)), hash);
|
||||
var casDirectory = GetCasDirectory(hash);
|
||||
Directory.CreateDirectory(casDirectory);
|
||||
|
||||
var fileName = SanitizeFileName(string.IsNullOrWhiteSpace(request.FileName) ? "callgraph.json" : request.FileName);
|
||||
var fileName = SanitizeFileName(string.IsNullOrWhiteSpace(request.FileName) ? DefaultFileName : request.FileName);
|
||||
var destinationPath = Path.Combine(casDirectory, fileName);
|
||||
|
||||
await using (var fileStream = File.Create(destinationPath))
|
||||
@@ -48,7 +50,7 @@ internal sealed class FileSystemCallgraphArtifactStore : ICallgraphArtifactStore
|
||||
await content.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var manifestPath = Path.Combine(casDirectory, "manifest.json");
|
||||
var manifestPath = Path.Combine(casDirectory, ManifestFileName);
|
||||
if (request.ManifestContent != null)
|
||||
{
|
||||
await using var manifestStream = File.Create(manifestPath);
|
||||
@@ -61,7 +63,7 @@ internal sealed class FileSystemCallgraphArtifactStore : ICallgraphArtifactStore
|
||||
}
|
||||
|
||||
var fileInfo = new FileInfo(destinationPath);
|
||||
logger.LogInformation("Stored callgraph artifact at {Path} (length={Length}).", destinationPath, fileInfo.Length);
|
||||
_logger.LogInformation("Stored callgraph artifact at {Path} (length={Length}).", destinationPath, fileInfo.Length);
|
||||
|
||||
return new StoredCallgraphArtifact(
|
||||
Path.GetRelativePath(root, destinationPath),
|
||||
@@ -73,6 +75,88 @@ internal sealed class FileSystemCallgraphArtifactStore : ICallgraphArtifactStore
|
||||
$"cas://reachability/graphs/{hash}/manifest");
|
||||
}
|
||||
|
||||
public Task<Stream?> GetAsync(string hash, string? fileName = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
var targetFileName = SanitizeFileName(string.IsNullOrWhiteSpace(fileName) ? DefaultFileName : fileName);
|
||||
var filePath = Path.Combine(casDirectory, targetFileName);
|
||||
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
_logger.LogDebug("Callgraph artifact {Hash}/{FileName} not found at {Path}.", normalizedHash, targetFileName, filePath);
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
var content = new MemoryStream();
|
||||
using (var fileStream = File.OpenRead(filePath))
|
||||
{
|
||||
fileStream.CopyTo(content);
|
||||
}
|
||||
|
||||
content.Position = 0;
|
||||
_logger.LogDebug("Retrieved callgraph artifact {Hash}/{FileName} from {Path}.", normalizedHash, targetFileName, filePath);
|
||||
return Task.FromResult<Stream?>(content);
|
||||
}
|
||||
|
||||
public Task<Stream?> GetManifestAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
var manifestPath = Path.Combine(casDirectory, ManifestFileName);
|
||||
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
_logger.LogDebug("Callgraph manifest for {Hash} not found at {Path}.", normalizedHash, manifestPath);
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
var content = new MemoryStream();
|
||||
using (var fileStream = File.OpenRead(manifestPath))
|
||||
{
|
||||
fileStream.CopyTo(content);
|
||||
}
|
||||
|
||||
content.Position = 0;
|
||||
_logger.LogDebug("Retrieved callgraph manifest for {Hash} from {Path}.", normalizedHash, manifestPath);
|
||||
return Task.FromResult<Stream?>(content);
|
||||
}
|
||||
|
||||
public Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return Task.FromResult(false);
|
||||
}
|
||||
|
||||
var casDirectory = GetCasDirectory(normalizedHash);
|
||||
var defaultPath = Path.Combine(casDirectory, DefaultFileName);
|
||||
var exists = File.Exists(defaultPath);
|
||||
|
||||
_logger.LogDebug("Callgraph artifact {Hash} exists={Exists} at {Path}.", normalizedHash, exists, defaultPath);
|
||||
return Task.FromResult(exists);
|
||||
}
|
||||
|
||||
private string GetCasDirectory(string hash)
|
||||
{
|
||||
var prefix = hash.Length >= 2 ? hash[..2] : hash;
|
||||
return Path.Combine(_storageOptions.RootPath, "cas", "reachability", "graphs", prefix, hash);
|
||||
}
|
||||
|
||||
private static string? NormalizeHash(string? hash)
|
||||
=> hash?.Trim().ToLowerInvariant();
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
=> string.Join('_', value.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)).ToLowerInvariant();
|
||||
}
|
||||
|
||||
@@ -6,9 +6,41 @@ using StellaOps.Signals.Storage.Models;
|
||||
namespace StellaOps.Signals.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Persists raw callgraph artifacts.
|
||||
/// Persists and retrieves raw callgraph artifacts from content-addressable storage.
|
||||
/// </summary>
|
||||
public interface ICallgraphArtifactStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Stores a callgraph artifact.
|
||||
/// </summary>
|
||||
/// <param name="request">Metadata about the artifact to store.</param>
|
||||
/// <param name="content">The artifact content stream.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Information about the stored artifact.</returns>
|
||||
Task<StoredCallgraphArtifact> SaveAsync(CallgraphArtifactSaveRequest request, Stream content, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a callgraph artifact by its hash.
|
||||
/// </summary>
|
||||
/// <param name="hash">The SHA-256 hash of the artifact.</param>
|
||||
/// <param name="fileName">Optional file name (defaults to callgraph.json).</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The artifact content stream, or null if not found.</returns>
|
||||
Task<Stream?> GetAsync(string hash, string? fileName = null, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a callgraph manifest by artifact hash.
|
||||
/// </summary>
|
||||
/// <param name="hash">The SHA-256 hash of the artifact.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The manifest content stream, or null if not found.</returns>
|
||||
Task<Stream?> GetManifestAsync(string hash, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if an artifact exists.
|
||||
/// </summary>
|
||||
/// <param name="hash">The SHA-256 hash of the artifact.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>True if the artifact exists.</returns>
|
||||
Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Signals.Options;
|
||||
using StellaOps.Signals.Storage.Models;
|
||||
|
||||
namespace StellaOps.Signals.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Stores callgraph artifacts in RustFS (S3-compatible content-addressable storage).
|
||||
/// </summary>
|
||||
internal sealed class RustFsCallgraphArtifactStore : ICallgraphArtifactStore
|
||||
{
|
||||
internal const string HttpClientName = "signals-storage-rustfs";
|
||||
|
||||
private const string DefaultFileName = "callgraph.json";
|
||||
private const string ManifestFileName = "manifest.json";
|
||||
private const string ImmutableHeader = "X-RustFS-Immutable";
|
||||
private const string RetainSecondsHeader = "X-RustFS-Retain-Seconds";
|
||||
private static readonly MediaTypeHeaderValue OctetStream = new("application/octet-stream");
|
||||
|
||||
/// <summary>
|
||||
/// Default retention for callgraph artifacts (90 days per CAS contract).
|
||||
/// </summary>
|
||||
private static readonly TimeSpan DefaultRetention = TimeSpan.FromDays(90);
|
||||
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly SignalsArtifactStorageOptions _storageOptions;
|
||||
private readonly ILogger<RustFsCallgraphArtifactStore> _logger;
|
||||
|
||||
public RustFsCallgraphArtifactStore(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<SignalsOptions> options,
|
||||
ILogger<RustFsCallgraphArtifactStore> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
_storageOptions = options.Value.Storage;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<StoredCallgraphArtifact> SaveAsync(CallgraphArtifactSaveRequest request, Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentNullException.ThrowIfNull(content);
|
||||
|
||||
var hash = NormalizeHash(request.Hash);
|
||||
if (string.IsNullOrWhiteSpace(hash))
|
||||
{
|
||||
throw new InvalidOperationException("Callgraph artifact hash is required for CAS storage.");
|
||||
}
|
||||
|
||||
var fileName = SanitizeFileName(string.IsNullOrWhiteSpace(request.FileName) ? DefaultFileName : request.FileName);
|
||||
var objectKey = BuildObjectKey(hash, fileName);
|
||||
|
||||
// Store the artifact
|
||||
await PutObjectAsync(objectKey, content, request.ContentType, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Store the manifest
|
||||
var manifestKey = BuildObjectKey(hash, ManifestFileName);
|
||||
if (request.ManifestContent != null)
|
||||
{
|
||||
request.ManifestContent.Position = 0;
|
||||
await PutObjectAsync(manifestKey, request.ManifestContent, "application/json", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Create empty manifest placeholder
|
||||
using var emptyManifest = new MemoryStream(Encoding.UTF8.GetBytes("{}"));
|
||||
await PutObjectAsync(manifestKey, emptyManifest, "application/json", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var artifactLength = content.CanSeek ? content.Length : 0;
|
||||
_logger.LogInformation("Stored callgraph artifact {Hash}/{FileName} in RustFS bucket {Bucket}.",
|
||||
hash, fileName, _storageOptions.BucketName);
|
||||
|
||||
return new StoredCallgraphArtifact(
|
||||
objectKey,
|
||||
artifactLength,
|
||||
hash,
|
||||
request.ContentType,
|
||||
$"cas://reachability/graphs/{hash}",
|
||||
manifestKey,
|
||||
$"cas://reachability/graphs/{hash}/manifest");
|
||||
}
|
||||
|
||||
public async Task<Stream?> GetAsync(string hash, string? fileName = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var targetFileName = SanitizeFileName(string.IsNullOrWhiteSpace(fileName) ? DefaultFileName : fileName);
|
||||
var objectKey = BuildObjectKey(normalizedHash, targetFileName);
|
||||
|
||||
var result = await GetObjectAsync(objectKey, cancellationToken).ConfigureAwait(false);
|
||||
if (result is null)
|
||||
{
|
||||
_logger.LogDebug("Callgraph artifact {Hash}/{FileName} not found in RustFS.", normalizedHash, targetFileName);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("Retrieved callgraph artifact {Hash}/{FileName} from RustFS.", normalizedHash, targetFileName);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<Stream?> GetManifestAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var manifestKey = BuildObjectKey(normalizedHash, ManifestFileName);
|
||||
var result = await GetObjectAsync(manifestKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (result is null)
|
||||
{
|
||||
_logger.LogDebug("Callgraph manifest for {Hash} not found in RustFS.", normalizedHash);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("Retrieved callgraph manifest for {Hash} from RustFS.", normalizedHash);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<bool> ExistsAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var normalizedHash = NormalizeHash(hash);
|
||||
if (string.IsNullOrWhiteSpace(normalizedHash))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var objectKey = BuildObjectKey(normalizedHash, DefaultFileName);
|
||||
var exists = await HeadObjectAsync(objectKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug("Callgraph artifact {Hash} exists={Exists} in RustFS.", normalizedHash, exists);
|
||||
return exists;
|
||||
}
|
||||
|
||||
private string BuildObjectKey(string hash, string fileName)
|
||||
{
|
||||
var prefix = hash.Length >= 2 ? hash[..2] : hash;
|
||||
var rootPrefix = string.IsNullOrWhiteSpace(_storageOptions.RootPrefix) ? "callgraphs" : _storageOptions.RootPrefix;
|
||||
return $"{rootPrefix}/{prefix}/{hash}/{fileName}";
|
||||
}
|
||||
|
||||
private async Task PutObjectAsync(string objectKey, Stream content, string? contentType, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(HttpClientName);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Put, BuildRequestUri(objectKey))
|
||||
{
|
||||
Content = CreateHttpContent(content)
|
||||
};
|
||||
|
||||
request.Content.Headers.ContentType = string.IsNullOrWhiteSpace(contentType)
|
||||
? OctetStream
|
||||
: new MediaTypeHeaderValue(contentType);
|
||||
|
||||
ApplyHeaders(request);
|
||||
|
||||
// Mark as immutable with 90-day retention per CAS contract
|
||||
request.Headers.TryAddWithoutValidation(ImmutableHeader, "true");
|
||||
var retainSeconds = Math.Ceiling(DefaultRetention.TotalSeconds);
|
||||
request.Headers.TryAddWithoutValidation(RetainSecondsHeader, retainSeconds.ToString(CultureInfo.InvariantCulture));
|
||||
|
||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var error = await ReadErrorAsync(response, cancellationToken).ConfigureAwait(false);
|
||||
throw new InvalidOperationException(
|
||||
$"RustFS upload for {_storageOptions.BucketName}/{objectKey} failed with status {(int)response.StatusCode} ({response.ReasonPhrase}). {error}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("Uploaded callgraph object {Bucket}/{Key} via RustFS.", _storageOptions.BucketName, objectKey);
|
||||
}
|
||||
|
||||
private async Task<Stream?> GetObjectAsync(string objectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(HttpClientName);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, BuildRequestUri(objectKey));
|
||||
ApplyHeaders(request);
|
||||
|
||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
if (response.StatusCode == HttpStatusCode.NotFound)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var error = await ReadErrorAsync(response, cancellationToken).ConfigureAwait(false);
|
||||
throw new InvalidOperationException(
|
||||
$"RustFS download for {_storageOptions.BucketName}/{objectKey} failed with status {(int)response.StatusCode} ({response.ReasonPhrase}). {error}");
|
||||
}
|
||||
|
||||
var buffer = new MemoryStream();
|
||||
if (response.Content is not null)
|
||||
{
|
||||
await response.Content.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
buffer.Position = 0;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
private async Task<bool> HeadObjectAsync(string objectKey, CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(HttpClientName);
|
||||
using var request = new HttpRequestMessage(HttpMethod.Head, BuildRequestUri(objectKey));
|
||||
ApplyHeaders(request);
|
||||
|
||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
return response.StatusCode == HttpStatusCode.OK;
|
||||
}
|
||||
|
||||
private Uri BuildRequestUri(string objectKey)
|
||||
{
|
||||
if (!Uri.TryCreate(_storageOptions.RustFs.BaseUrl, UriKind.Absolute, out var baseUri))
|
||||
{
|
||||
throw new InvalidOperationException("RustFS baseUrl is invalid.");
|
||||
}
|
||||
|
||||
var encodedBucket = Uri.EscapeDataString(_storageOptions.BucketName);
|
||||
var encodedKey = EncodeKey(objectKey);
|
||||
var relativePath = new StringBuilder()
|
||||
.Append("buckets/")
|
||||
.Append(encodedBucket)
|
||||
.Append("/objects/")
|
||||
.Append(encodedKey)
|
||||
.ToString();
|
||||
|
||||
return new Uri(baseUri, relativePath);
|
||||
}
|
||||
|
||||
private static string EncodeKey(string key)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(key))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var segments = key.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
return string.Join('/', segments.Select(Uri.EscapeDataString));
|
||||
}
|
||||
|
||||
private void ApplyHeaders(HttpRequestMessage request)
|
||||
{
|
||||
var rustFsOptions = _storageOptions.RustFs;
|
||||
if (!string.IsNullOrWhiteSpace(rustFsOptions.ApiKeyHeader) && !string.IsNullOrWhiteSpace(rustFsOptions.ApiKey))
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(rustFsOptions.ApiKeyHeader, rustFsOptions.ApiKey);
|
||||
}
|
||||
|
||||
foreach (var header in _storageOptions.Headers)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpContent CreateHttpContent(Stream content)
|
||||
{
|
||||
if (content is MemoryStream memoryStream)
|
||||
{
|
||||
if (memoryStream.TryGetBuffer(out var segment))
|
||||
{
|
||||
return new ByteArrayContent(segment.Array!, segment.Offset, segment.Count);
|
||||
}
|
||||
|
||||
return new ByteArrayContent(memoryStream.ToArray());
|
||||
}
|
||||
|
||||
if (content.CanSeek)
|
||||
{
|
||||
var originalPosition = content.Position;
|
||||
try
|
||||
{
|
||||
content.Position = 0;
|
||||
using var duplicate = new MemoryStream();
|
||||
content.CopyTo(duplicate);
|
||||
return new ByteArrayContent(duplicate.ToArray());
|
||||
}
|
||||
finally
|
||||
{
|
||||
content.Position = originalPosition;
|
||||
}
|
||||
}
|
||||
|
||||
using var buffer = new MemoryStream();
|
||||
content.CopyTo(buffer);
|
||||
return new ByteArrayContent(buffer.ToArray());
|
||||
}
|
||||
|
||||
private static async Task<string> ReadErrorAsync(HttpResponseMessage response, CancellationToken cancellationToken)
|
||||
{
|
||||
if (response.Content is null)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = text.Trim();
|
||||
return trimmed.Length <= 512 ? trimmed : trimmed[..512];
|
||||
}
|
||||
|
||||
private static string? NormalizeHash(string? hash)
|
||||
=> hash?.Trim().ToLowerInvariant();
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
=> string.Join('_', value.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)).ToLowerInvariant();
|
||||
}
|
||||
Reference in New Issue
Block a user