using StellaOps.Determinism; using StellaOps.VexLens.Consensus; using StellaOps.VexLens.Models; using StellaOps.VexLens.Storage; using System.Text.Json; namespace StellaOps.VexLens.Export; /// /// Service for exporting consensus projections to offline bundles. /// public interface IConsensusExportService { /// /// Creates a snapshot of consensus projections. /// Task CreateSnapshotAsync( SnapshotRequest request, CancellationToken cancellationToken = default); /// /// Exports snapshot to a stream in the specified format. /// Task ExportToStreamAsync( ConsensusSnapshot snapshot, Stream outputStream, ExportFormat format = ExportFormat.JsonLines, CancellationToken cancellationToken = default); /// /// Creates an incremental snapshot since the last export. /// Task CreateIncrementalSnapshotAsync( string? lastSnapshotId, DateTimeOffset? since, SnapshotRequest request, CancellationToken cancellationToken = default); /// /// Verifies a snapshot against stored projections. /// Task VerifySnapshotAsync( ConsensusSnapshot snapshot, CancellationToken cancellationToken = default); } /// /// Request for creating a snapshot. /// public sealed record SnapshotRequest( /// /// Tenant ID filter (null for all tenants). /// string? TenantId, /// /// Filter by vulnerability IDs (null for all). /// IReadOnlyList? VulnerabilityIds, /// /// Filter by product keys (null for all). /// IReadOnlyList? ProductKeys, /// /// Minimum confidence threshold. /// double? MinimumConfidence, /// /// Filter by status (null for all). /// VexStatus? Status, /// /// Include projections computed after this time. /// DateTimeOffset? ComputedAfter, /// /// Include projections computed before this time. /// DateTimeOffset? ComputedBefore, /// /// Include projection history. /// bool IncludeHistory, /// /// Maximum projections to include. /// int? MaxProjections); /// /// A snapshot of consensus projections. /// public sealed record ConsensusSnapshot( /// /// Unique snapshot identifier. /// string SnapshotId, /// /// When the snapshot was created. /// DateTimeOffset CreatedAt, /// /// Snapshot version for format compatibility. /// string Version, /// /// Tenant ID if filtered. /// string? TenantId, /// /// The consensus projections. /// IReadOnlyList Projections, /// /// Projection history if requested. /// IReadOnlyList? History, /// /// Snapshot metadata. /// SnapshotMetadata Metadata); /// /// Metadata about a snapshot. /// public sealed record SnapshotMetadata( /// /// Total projections in snapshot. /// int TotalProjections, /// /// Total history entries if included. /// int TotalHistoryEntries, /// /// Oldest projection in snapshot. /// DateTimeOffset? OldestProjection, /// /// Newest projection in snapshot. /// DateTimeOffset? NewestProjection, /// /// Status counts. /// IReadOnlyDictionary StatusCounts, /// /// Content hash for verification. /// string ContentHash, /// /// Creator identifier. /// string? CreatedBy); /// /// Incremental snapshot since last export. /// public sealed record IncrementalSnapshot( /// /// This snapshot's ID. /// string SnapshotId, /// /// Previous snapshot ID this is based on. /// string? PreviousSnapshotId, /// /// When the snapshot was created. /// DateTimeOffset CreatedAt, /// /// Snapshot version. /// string Version, /// /// New or updated projections. /// IReadOnlyList Added, /// /// Removed projection keys. /// IReadOnlyList Removed, /// /// Incremental metadata. /// IncrementalMetadata Metadata); /// /// Key identifying a projection. /// public sealed record ProjectionKey( string VulnerabilityId, string ProductKey, string? TenantId); /// /// Metadata for incremental snapshot. /// public sealed record IncrementalMetadata( int AddedCount, int RemovedCount, DateTimeOffset? SinceTimestamp, string ContentHash); /// /// Result of snapshot verification. /// public sealed record SnapshotVerificationResult( bool IsValid, string? ErrorMessage, int VerifiedCount, int MismatchCount, IReadOnlyList? Mismatches); /// /// A mismatch found during verification. /// public sealed record VerificationMismatch( string VulnerabilityId, string ProductKey, string Field, string? ExpectedValue, string? ActualValue); /// /// Export format. /// public enum ExportFormat { /// /// NDJSON (newline-delimited JSON). /// JsonLines, /// /// Single JSON document. /// Json, /// /// Compact binary format. /// Binary } /// /// Default implementation of . /// public sealed class ConsensusExportService : IConsensusExportService { private readonly IConsensusProjectionStore _projectionStore; private readonly TimeProvider _timeProvider; private readonly IGuidProvider _guidProvider; private const string SnapshotVersion = "1.0.0"; public ConsensusExportService( IConsensusProjectionStore projectionStore, TimeProvider? timeProvider = null, IGuidProvider? guidProvider = null) { _projectionStore = projectionStore; _timeProvider = timeProvider ?? TimeProvider.System; _guidProvider = guidProvider ?? new SystemGuidProvider(); } public async Task CreateSnapshotAsync( SnapshotRequest request, CancellationToken cancellationToken = default) { var query = new ProjectionQuery( TenantId: request.TenantId, VulnerabilityId: request.VulnerabilityIds?.FirstOrDefault(), ProductKey: request.ProductKeys?.FirstOrDefault(), Status: request.Status, Outcome: null, MinimumConfidence: request.MinimumConfidence, ComputedAfter: request.ComputedAfter, ComputedBefore: request.ComputedBefore, StatusChanged: null, Limit: request.MaxProjections ?? 10000, Offset: 0, SortBy: ProjectionSortField.ComputedAt, SortDescending: true); var result = await _projectionStore.ListAsync(query, cancellationToken); // Filter by additional criteria if needed var projections = result.Projections.ToList(); if (request.VulnerabilityIds is { Count: > 1 }) { var vulnSet = new HashSet(request.VulnerabilityIds); projections = projections.Where(p => vulnSet.Contains(p.VulnerabilityId)).ToList(); } if (request.ProductKeys is { Count: > 1 }) { var productSet = new HashSet(request.ProductKeys); projections = projections.Where(p => productSet.Contains(p.ProductKey)).ToList(); } // Load history if requested List? history = null; if (request.IncludeHistory) { history = []; foreach (var projection in projections.Take(100)) // Limit history loading { var projHistory = await _projectionStore.GetHistoryAsync( projection.VulnerabilityId, projection.ProductKey, projection.TenantId, 10, cancellationToken); history.AddRange(projHistory); } } var statusCounts = projections .GroupBy(p => p.Status) .ToDictionary(g => g.Key, g => g.Count()); var snapshotId = $"snap-{_guidProvider.NewGuid():N}"; var contentHash = ComputeContentHash(projections); return new ConsensusSnapshot( SnapshotId: snapshotId, CreatedAt: _timeProvider.GetUtcNow(), Version: SnapshotVersion, TenantId: request.TenantId, Projections: projections, History: history, Metadata: new SnapshotMetadata( TotalProjections: projections.Count, TotalHistoryEntries: history?.Count ?? 0, OldestProjection: projections.Min(p => (DateTimeOffset?)p.ComputedAt), NewestProjection: projections.Max(p => (DateTimeOffset?)p.ComputedAt), StatusCounts: statusCounts, ContentHash: contentHash, CreatedBy: "VexLens")); } public async Task ExportToStreamAsync( ConsensusSnapshot snapshot, Stream outputStream, ExportFormat format = ExportFormat.JsonLines, CancellationToken cancellationToken = default) { var options = new JsonSerializerOptions { WriteIndented = format == ExportFormat.Json, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; switch (format) { case ExportFormat.JsonLines: await ExportAsJsonLinesAsync(snapshot, outputStream, options, cancellationToken); break; case ExportFormat.Json: await JsonSerializer.SerializeAsync(outputStream, snapshot, options, cancellationToken); break; case ExportFormat.Binary: // For binary format, use JSON with no indentation as a simple binary-ish format options.WriteIndented = false; await JsonSerializer.SerializeAsync(outputStream, snapshot, options, cancellationToken); break; } } public async Task CreateIncrementalSnapshotAsync( string? lastSnapshotId, DateTimeOffset? since, SnapshotRequest request, CancellationToken cancellationToken = default) { // Get current projections var currentRequest = request with { ComputedAfter = since }; var current = await CreateSnapshotAsync(currentRequest, cancellationToken); // For a true incremental, we'd compare with the previous snapshot // Here we just return new/updated since the timestamp var snapshotId = $"snap-inc-{_guidProvider.NewGuid():N}"; var contentHash = ComputeContentHash(current.Projections); return new IncrementalSnapshot( SnapshotId: snapshotId, PreviousSnapshotId: lastSnapshotId, CreatedAt: _timeProvider.GetUtcNow(), Version: SnapshotVersion, Added: current.Projections, Removed: [], // Would need previous snapshot to determine removed Metadata: new IncrementalMetadata( AddedCount: current.Projections.Count, RemovedCount: 0, SinceTimestamp: since, ContentHash: contentHash)); } public async Task VerifySnapshotAsync( ConsensusSnapshot snapshot, CancellationToken cancellationToken = default) { var mismatches = new List(); var verifiedCount = 0; foreach (var projection in snapshot.Projections) { var current = await _projectionStore.GetLatestAsync( projection.VulnerabilityId, projection.ProductKey, projection.TenantId, cancellationToken); if (current == null) { mismatches.Add(new VerificationMismatch( projection.VulnerabilityId, projection.ProductKey, "existence", "exists", "not found")); continue; } // Check key fields if (current.Status != projection.Status) { mismatches.Add(new VerificationMismatch( projection.VulnerabilityId, projection.ProductKey, "status", projection.Status.ToString(), current.Status.ToString())); } if (Math.Abs(current.ConfidenceScore - projection.ConfidenceScore) > 0.001) { mismatches.Add(new VerificationMismatch( projection.VulnerabilityId, projection.ProductKey, "confidenceScore", projection.ConfidenceScore.ToString("F4"), current.ConfidenceScore.ToString("F4"))); } verifiedCount++; } return new SnapshotVerificationResult( IsValid: mismatches.Count == 0, ErrorMessage: mismatches.Count > 0 ? $"{mismatches.Count} mismatch(es) found" : null, VerifiedCount: verifiedCount, MismatchCount: mismatches.Count, Mismatches: mismatches.Count > 0 ? mismatches : null); } private static async Task ExportAsJsonLinesAsync( ConsensusSnapshot snapshot, Stream outputStream, JsonSerializerOptions options, CancellationToken cancellationToken) { await using var writer = new StreamWriter(outputStream, leaveOpen: true); // Write header line var header = new { type = "header", snapshotId = snapshot.SnapshotId, createdAt = snapshot.CreatedAt, version = snapshot.Version, metadata = snapshot.Metadata }; await writer.WriteLineAsync(JsonSerializer.Serialize(header, options)); // Write each projection foreach (var projection in snapshot.Projections) { cancellationToken.ThrowIfCancellationRequested(); var line = new { type = "projection", data = projection }; await writer.WriteLineAsync(JsonSerializer.Serialize(line, options)); } // Write history if present if (snapshot.History != null) { foreach (var historyEntry in snapshot.History) { cancellationToken.ThrowIfCancellationRequested(); var line = new { type = "history", data = historyEntry }; await writer.WriteLineAsync(JsonSerializer.Serialize(line, options)); } } // Write footer var footer = new { type = "footer", totalProjections = snapshot.Projections.Count, totalHistory = snapshot.History?.Count ?? 0, contentHash = snapshot.Metadata.ContentHash }; await writer.WriteLineAsync(JsonSerializer.Serialize(footer, options)); } private static string ComputeContentHash(IReadOnlyList projections) { var data = string.Join("|", projections .OrderBy(p => p.VulnerabilityId) .ThenBy(p => p.ProductKey) .Select(p => $"{p.VulnerabilityId}:{p.ProductKey}:{p.Status}:{p.ConfidenceScore:F4}")); var hash = System.Security.Cryptography.SHA256.HashData( System.Text.Encoding.UTF8.GetBytes(data)); return Convert.ToHexString(hash).ToLowerInvariant()[..32]; } } /// /// Extensions for export configuration. /// public static class ConsensusExportExtensions { /// /// Creates a snapshot request for full export. /// public static SnapshotRequest FullExportRequest(string? tenantId = null) { return new SnapshotRequest( TenantId: tenantId, VulnerabilityIds: null, ProductKeys: null, MinimumConfidence: null, Status: null, ComputedAfter: null, ComputedBefore: null, IncludeHistory: false, MaxProjections: null); } /// /// Creates a snapshot request for mirror bundle export. /// public static SnapshotRequest MirrorBundleRequest( string? tenantId = null, double minimumConfidence = 0.5, bool includeHistory = false) { return new SnapshotRequest( TenantId: tenantId, VulnerabilityIds: null, ProductKeys: null, MinimumConfidence: minimumConfidence, Status: null, ComputedAfter: null, ComputedBefore: null, IncludeHistory: includeHistory, MaxProjections: 100000); } }