using StellaOps.Determinism;
using StellaOps.VexLens.Consensus;
using StellaOps.VexLens.Models;
using StellaOps.VexLens.Storage;
using System.Text.Json;
namespace StellaOps.VexLens.Export;
///
/// Service for exporting consensus projections to offline bundles.
///
public interface IConsensusExportService
{
///
/// Creates a snapshot of consensus projections.
///
Task CreateSnapshotAsync(
SnapshotRequest request,
CancellationToken cancellationToken = default);
///
/// Exports snapshot to a stream in the specified format.
///
Task ExportToStreamAsync(
ConsensusSnapshot snapshot,
Stream outputStream,
ExportFormat format = ExportFormat.JsonLines,
CancellationToken cancellationToken = default);
///
/// Creates an incremental snapshot since the last export.
///
Task CreateIncrementalSnapshotAsync(
string? lastSnapshotId,
DateTimeOffset? since,
SnapshotRequest request,
CancellationToken cancellationToken = default);
///
/// Verifies a snapshot against stored projections.
///
Task VerifySnapshotAsync(
ConsensusSnapshot snapshot,
CancellationToken cancellationToken = default);
}
///
/// Request for creating a snapshot.
///
public sealed record SnapshotRequest(
///
/// Tenant ID filter (null for all tenants).
///
string? TenantId,
///
/// Filter by vulnerability IDs (null for all).
///
IReadOnlyList? VulnerabilityIds,
///
/// Filter by product keys (null for all).
///
IReadOnlyList? ProductKeys,
///
/// Minimum confidence threshold.
///
double? MinimumConfidence,
///
/// Filter by status (null for all).
///
VexStatus? Status,
///
/// Include projections computed after this time.
///
DateTimeOffset? ComputedAfter,
///
/// Include projections computed before this time.
///
DateTimeOffset? ComputedBefore,
///
/// Include projection history.
///
bool IncludeHistory,
///
/// Maximum projections to include.
///
int? MaxProjections);
///
/// A snapshot of consensus projections.
///
public sealed record ConsensusSnapshot(
///
/// Unique snapshot identifier.
///
string SnapshotId,
///
/// When the snapshot was created.
///
DateTimeOffset CreatedAt,
///
/// Snapshot version for format compatibility.
///
string Version,
///
/// Tenant ID if filtered.
///
string? TenantId,
///
/// The consensus projections.
///
IReadOnlyList Projections,
///
/// Projection history if requested.
///
IReadOnlyList? History,
///
/// Snapshot metadata.
///
SnapshotMetadata Metadata);
///
/// Metadata about a snapshot.
///
public sealed record SnapshotMetadata(
///
/// Total projections in snapshot.
///
int TotalProjections,
///
/// Total history entries if included.
///
int TotalHistoryEntries,
///
/// Oldest projection in snapshot.
///
DateTimeOffset? OldestProjection,
///
/// Newest projection in snapshot.
///
DateTimeOffset? NewestProjection,
///
/// Status counts.
///
IReadOnlyDictionary StatusCounts,
///
/// Content hash for verification.
///
string ContentHash,
///
/// Creator identifier.
///
string? CreatedBy);
///
/// Incremental snapshot since last export.
///
public sealed record IncrementalSnapshot(
///
/// This snapshot's ID.
///
string SnapshotId,
///
/// Previous snapshot ID this is based on.
///
string? PreviousSnapshotId,
///
/// When the snapshot was created.
///
DateTimeOffset CreatedAt,
///
/// Snapshot version.
///
string Version,
///
/// New or updated projections.
///
IReadOnlyList Added,
///
/// Removed projection keys.
///
IReadOnlyList Removed,
///
/// Incremental metadata.
///
IncrementalMetadata Metadata);
///
/// Key identifying a projection.
///
public sealed record ProjectionKey(
string VulnerabilityId,
string ProductKey,
string? TenantId);
///
/// Metadata for incremental snapshot.
///
public sealed record IncrementalMetadata(
int AddedCount,
int RemovedCount,
DateTimeOffset? SinceTimestamp,
string ContentHash);
///
/// Result of snapshot verification.
///
public sealed record SnapshotVerificationResult(
bool IsValid,
string? ErrorMessage,
int VerifiedCount,
int MismatchCount,
IReadOnlyList? Mismatches);
///
/// A mismatch found during verification.
///
public sealed record VerificationMismatch(
string VulnerabilityId,
string ProductKey,
string Field,
string? ExpectedValue,
string? ActualValue);
///
/// Export format.
///
public enum ExportFormat
{
///
/// NDJSON (newline-delimited JSON).
///
JsonLines,
///
/// Single JSON document.
///
Json,
///
/// Compact binary format.
///
Binary
}
///
/// Default implementation of .
///
public sealed class ConsensusExportService : IConsensusExportService
{
private readonly IConsensusProjectionStore _projectionStore;
private readonly TimeProvider _timeProvider;
private readonly IGuidProvider _guidProvider;
private const string SnapshotVersion = "1.0.0";
public ConsensusExportService(
IConsensusProjectionStore projectionStore,
TimeProvider? timeProvider = null,
IGuidProvider? guidProvider = null)
{
_projectionStore = projectionStore;
_timeProvider = timeProvider ?? TimeProvider.System;
_guidProvider = guidProvider ?? new SystemGuidProvider();
}
public async Task CreateSnapshotAsync(
SnapshotRequest request,
CancellationToken cancellationToken = default)
{
var query = new ProjectionQuery(
TenantId: request.TenantId,
VulnerabilityId: request.VulnerabilityIds?.FirstOrDefault(),
ProductKey: request.ProductKeys?.FirstOrDefault(),
Status: request.Status,
Outcome: null,
MinimumConfidence: request.MinimumConfidence,
ComputedAfter: request.ComputedAfter,
ComputedBefore: request.ComputedBefore,
StatusChanged: null,
Limit: request.MaxProjections ?? 10000,
Offset: 0,
SortBy: ProjectionSortField.ComputedAt,
SortDescending: true);
var result = await _projectionStore.ListAsync(query, cancellationToken);
// Filter by additional criteria if needed
var projections = result.Projections.ToList();
if (request.VulnerabilityIds is { Count: > 1 })
{
var vulnSet = new HashSet(request.VulnerabilityIds);
projections = projections.Where(p => vulnSet.Contains(p.VulnerabilityId)).ToList();
}
if (request.ProductKeys is { Count: > 1 })
{
var productSet = new HashSet(request.ProductKeys);
projections = projections.Where(p => productSet.Contains(p.ProductKey)).ToList();
}
// Load history if requested
List? history = null;
if (request.IncludeHistory)
{
history = [];
foreach (var projection in projections.Take(100)) // Limit history loading
{
var projHistory = await _projectionStore.GetHistoryAsync(
projection.VulnerabilityId,
projection.ProductKey,
projection.TenantId,
10,
cancellationToken);
history.AddRange(projHistory);
}
}
var statusCounts = projections
.GroupBy(p => p.Status)
.ToDictionary(g => g.Key, g => g.Count());
var snapshotId = $"snap-{_guidProvider.NewGuid():N}";
var contentHash = ComputeContentHash(projections);
return new ConsensusSnapshot(
SnapshotId: snapshotId,
CreatedAt: _timeProvider.GetUtcNow(),
Version: SnapshotVersion,
TenantId: request.TenantId,
Projections: projections,
History: history,
Metadata: new SnapshotMetadata(
TotalProjections: projections.Count,
TotalHistoryEntries: history?.Count ?? 0,
OldestProjection: projections.Min(p => (DateTimeOffset?)p.ComputedAt),
NewestProjection: projections.Max(p => (DateTimeOffset?)p.ComputedAt),
StatusCounts: statusCounts,
ContentHash: contentHash,
CreatedBy: "VexLens"));
}
public async Task ExportToStreamAsync(
ConsensusSnapshot snapshot,
Stream outputStream,
ExportFormat format = ExportFormat.JsonLines,
CancellationToken cancellationToken = default)
{
var options = new JsonSerializerOptions
{
WriteIndented = format == ExportFormat.Json,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
};
switch (format)
{
case ExportFormat.JsonLines:
await ExportAsJsonLinesAsync(snapshot, outputStream, options, cancellationToken);
break;
case ExportFormat.Json:
await JsonSerializer.SerializeAsync(outputStream, snapshot, options, cancellationToken);
break;
case ExportFormat.Binary:
// For binary format, use JSON with no indentation as a simple binary-ish format
options.WriteIndented = false;
await JsonSerializer.SerializeAsync(outputStream, snapshot, options, cancellationToken);
break;
}
}
public async Task CreateIncrementalSnapshotAsync(
string? lastSnapshotId,
DateTimeOffset? since,
SnapshotRequest request,
CancellationToken cancellationToken = default)
{
// Get current projections
var currentRequest = request with { ComputedAfter = since };
var current = await CreateSnapshotAsync(currentRequest, cancellationToken);
// For a true incremental, we'd compare with the previous snapshot
// Here we just return new/updated since the timestamp
var snapshotId = $"snap-inc-{_guidProvider.NewGuid():N}";
var contentHash = ComputeContentHash(current.Projections);
return new IncrementalSnapshot(
SnapshotId: snapshotId,
PreviousSnapshotId: lastSnapshotId,
CreatedAt: _timeProvider.GetUtcNow(),
Version: SnapshotVersion,
Added: current.Projections,
Removed: [], // Would need previous snapshot to determine removed
Metadata: new IncrementalMetadata(
AddedCount: current.Projections.Count,
RemovedCount: 0,
SinceTimestamp: since,
ContentHash: contentHash));
}
public async Task VerifySnapshotAsync(
ConsensusSnapshot snapshot,
CancellationToken cancellationToken = default)
{
var mismatches = new List();
var verifiedCount = 0;
foreach (var projection in snapshot.Projections)
{
var current = await _projectionStore.GetLatestAsync(
projection.VulnerabilityId,
projection.ProductKey,
projection.TenantId,
cancellationToken);
if (current == null)
{
mismatches.Add(new VerificationMismatch(
projection.VulnerabilityId,
projection.ProductKey,
"existence",
"exists",
"not found"));
continue;
}
// Check key fields
if (current.Status != projection.Status)
{
mismatches.Add(new VerificationMismatch(
projection.VulnerabilityId,
projection.ProductKey,
"status",
projection.Status.ToString(),
current.Status.ToString()));
}
if (Math.Abs(current.ConfidenceScore - projection.ConfidenceScore) > 0.001)
{
mismatches.Add(new VerificationMismatch(
projection.VulnerabilityId,
projection.ProductKey,
"confidenceScore",
projection.ConfidenceScore.ToString("F4"),
current.ConfidenceScore.ToString("F4")));
}
verifiedCount++;
}
return new SnapshotVerificationResult(
IsValid: mismatches.Count == 0,
ErrorMessage: mismatches.Count > 0 ? $"{mismatches.Count} mismatch(es) found" : null,
VerifiedCount: verifiedCount,
MismatchCount: mismatches.Count,
Mismatches: mismatches.Count > 0 ? mismatches : null);
}
private static async Task ExportAsJsonLinesAsync(
ConsensusSnapshot snapshot,
Stream outputStream,
JsonSerializerOptions options,
CancellationToken cancellationToken)
{
await using var writer = new StreamWriter(outputStream, leaveOpen: true);
// Write header line
var header = new
{
type = "header",
snapshotId = snapshot.SnapshotId,
createdAt = snapshot.CreatedAt,
version = snapshot.Version,
metadata = snapshot.Metadata
};
await writer.WriteLineAsync(JsonSerializer.Serialize(header, options));
// Write each projection
foreach (var projection in snapshot.Projections)
{
cancellationToken.ThrowIfCancellationRequested();
var line = new { type = "projection", data = projection };
await writer.WriteLineAsync(JsonSerializer.Serialize(line, options));
}
// Write history if present
if (snapshot.History != null)
{
foreach (var historyEntry in snapshot.History)
{
cancellationToken.ThrowIfCancellationRequested();
var line = new { type = "history", data = historyEntry };
await writer.WriteLineAsync(JsonSerializer.Serialize(line, options));
}
}
// Write footer
var footer = new
{
type = "footer",
totalProjections = snapshot.Projections.Count,
totalHistory = snapshot.History?.Count ?? 0,
contentHash = snapshot.Metadata.ContentHash
};
await writer.WriteLineAsync(JsonSerializer.Serialize(footer, options));
}
private static string ComputeContentHash(IReadOnlyList projections)
{
var data = string.Join("|", projections
.OrderBy(p => p.VulnerabilityId)
.ThenBy(p => p.ProductKey)
.Select(p => $"{p.VulnerabilityId}:{p.ProductKey}:{p.Status}:{p.ConfidenceScore:F4}"));
var hash = System.Security.Cryptography.SHA256.HashData(
System.Text.Encoding.UTF8.GetBytes(data));
return Convert.ToHexString(hash).ToLowerInvariant()[..32];
}
}
///
/// Extensions for export configuration.
///
public static class ConsensusExportExtensions
{
///
/// Creates a snapshot request for full export.
///
public static SnapshotRequest FullExportRequest(string? tenantId = null)
{
return new SnapshotRequest(
TenantId: tenantId,
VulnerabilityIds: null,
ProductKeys: null,
MinimumConfidence: null,
Status: null,
ComputedAfter: null,
ComputedBefore: null,
IncludeHistory: false,
MaxProjections: null);
}
///
/// Creates a snapshot request for mirror bundle export.
///
public static SnapshotRequest MirrorBundleRequest(
string? tenantId = null,
double minimumConfidence = 0.5,
bool includeHistory = false)
{
return new SnapshotRequest(
TenantId: tenantId,
VulnerabilityIds: null,
ProductKeys: null,
MinimumConfidence: minimumConfidence,
Status: null,
ComputedAfter: null,
ComputedBefore: null,
IncludeHistory: includeHistory,
MaxProjections: 100000);
}
}