Add unit tests for VexLens normalizer, CPE parser, product mapper, and PURL parser
- Implemented comprehensive tests for VexLensNormalizer including format detection and normalization scenarios. - Added tests for CpeParser covering CPE 2.3 and 2.2 formats, invalid inputs, and canonical key generation. - Created tests for ProductMapper to validate parsing and matching logic across different strictness levels. - Developed tests for PurlParser to ensure correct parsing of various PURL formats and validation of identifiers. - Introduced stubs for Monaco editor and worker to facilitate testing in the web application. - Updated project file for the test project to include necessary dependencies.
This commit is contained in:
StellaOps Bot
581
src/VexLens/StellaOps.VexLens/Export/IConsensusExportService.cs
Normal file
581
src/VexLens/StellaOps.VexLens/Export/IConsensusExportService.cs
Normal file
@@ -0,0 +1,581 @@
|
||||
using System.Text.Json;
|
||||
using StellaOps.VexLens.Consensus;
|
||||
using StellaOps.VexLens.Models;
|
||||
using StellaOps.VexLens.Storage;
|
||||
|
||||
namespace StellaOps.VexLens.Export;
|
||||
|
||||
/// <summary>
|
||||
/// Service for exporting consensus projections to offline bundles.
|
||||
/// </summary>
|
||||
public interface IConsensusExportService
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a snapshot of consensus projections.
|
||||
/// </summary>
|
||||
Task<ConsensusSnapshot> CreateSnapshotAsync(
|
||||
SnapshotRequest request,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Exports snapshot to a stream in the specified format.
|
||||
/// </summary>
|
||||
Task ExportToStreamAsync(
|
||||
ConsensusSnapshot snapshot,
|
||||
Stream outputStream,
|
||||
ExportFormat format = ExportFormat.JsonLines,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Creates an incremental snapshot since the last export.
|
||||
/// </summary>
|
||||
Task<IncrementalSnapshot> CreateIncrementalSnapshotAsync(
|
||||
string? lastSnapshotId,
|
||||
DateTimeOffset? since,
|
||||
SnapshotRequest request,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a snapshot against stored projections.
|
||||
/// </summary>
|
||||
Task<SnapshotVerificationResult> VerifySnapshotAsync(
|
||||
ConsensusSnapshot snapshot,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Request for creating a snapshot.
|
||||
/// </summary>
|
||||
public sealed record SnapshotRequest(
|
||||
/// <summary>
|
||||
/// Tenant ID filter (null for all tenants).
|
||||
/// </summary>
|
||||
string? TenantId,
|
||||
|
||||
/// <summary>
|
||||
/// Filter by vulnerability IDs (null for all).
|
||||
/// </summary>
|
||||
IReadOnlyList<string>? VulnerabilityIds,
|
||||
|
||||
/// <summary>
|
||||
/// Filter by product keys (null for all).
|
||||
/// </summary>
|
||||
IReadOnlyList<string>? ProductKeys,
|
||||
|
||||
/// <summary>
|
||||
/// Minimum confidence threshold.
|
||||
/// </summary>
|
||||
double? MinimumConfidence,
|
||||
|
||||
/// <summary>
|
||||
/// Filter by status (null for all).
|
||||
/// </summary>
|
||||
VexStatus? Status,
|
||||
|
||||
/// <summary>
|
||||
/// Include projections computed after this time.
|
||||
/// </summary>
|
||||
DateTimeOffset? ComputedAfter,
|
||||
|
||||
/// <summary>
|
||||
/// Include projections computed before this time.
|
||||
/// </summary>
|
||||
DateTimeOffset? ComputedBefore,
|
||||
|
||||
/// <summary>
|
||||
/// Include projection history.
|
||||
/// </summary>
|
||||
bool IncludeHistory,
|
||||
|
||||
/// <summary>
|
||||
/// Maximum projections to include.
|
||||
/// </summary>
|
||||
int? MaxProjections);
|
||||
|
||||
/// <summary>
|
||||
/// A snapshot of consensus projections.
|
||||
/// </summary>
|
||||
public sealed record ConsensusSnapshot(
|
||||
/// <summary>
|
||||
/// Unique snapshot identifier.
|
||||
/// </summary>
|
||||
string SnapshotId,
|
||||
|
||||
/// <summary>
|
||||
/// When the snapshot was created.
|
||||
/// </summary>
|
||||
DateTimeOffset CreatedAt,
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot version for format compatibility.
|
||||
/// </summary>
|
||||
string Version,
|
||||
|
||||
/// <summary>
|
||||
/// Tenant ID if filtered.
|
||||
/// </summary>
|
||||
string? TenantId,
|
||||
|
||||
/// <summary>
|
||||
/// The consensus projections.
|
||||
/// </summary>
|
||||
IReadOnlyList<ConsensusProjection> Projections,
|
||||
|
||||
/// <summary>
|
||||
/// Projection history if requested.
|
||||
/// </summary>
|
||||
IReadOnlyList<ConsensusProjection>? History,
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot metadata.
|
||||
/// </summary>
|
||||
SnapshotMetadata Metadata);
|
||||
|
||||
/// <summary>
|
||||
/// Metadata about a snapshot.
|
||||
/// </summary>
|
||||
public sealed record SnapshotMetadata(
|
||||
/// <summary>
|
||||
/// Total projections in snapshot.
|
||||
/// </summary>
|
||||
int TotalProjections,
|
||||
|
||||
/// <summary>
|
||||
/// Total history entries if included.
|
||||
/// </summary>
|
||||
int TotalHistoryEntries,
|
||||
|
||||
/// <summary>
|
||||
/// Oldest projection in snapshot.
|
||||
/// </summary>
|
||||
DateTimeOffset? OldestProjection,
|
||||
|
||||
/// <summary>
|
||||
/// Newest projection in snapshot.
|
||||
/// </summary>
|
||||
DateTimeOffset? NewestProjection,
|
||||
|
||||
/// <summary>
|
||||
/// Status counts.
|
||||
/// </summary>
|
||||
IReadOnlyDictionary<VexStatus, int> StatusCounts,
|
||||
|
||||
/// <summary>
|
||||
/// Content hash for verification.
|
||||
/// </summary>
|
||||
string ContentHash,
|
||||
|
||||
/// <summary>
|
||||
/// Creator identifier.
|
||||
/// </summary>
|
||||
string? CreatedBy);
|
||||
|
||||
/// <summary>
|
||||
/// Incremental snapshot since last export.
|
||||
/// </summary>
|
||||
public sealed record IncrementalSnapshot(
|
||||
/// <summary>
|
||||
/// This snapshot's ID.
|
||||
/// </summary>
|
||||
string SnapshotId,
|
||||
|
||||
/// <summary>
|
||||
/// Previous snapshot ID this is based on.
|
||||
/// </summary>
|
||||
string? PreviousSnapshotId,
|
||||
|
||||
/// <summary>
|
||||
/// When the snapshot was created.
|
||||
/// </summary>
|
||||
DateTimeOffset CreatedAt,
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot version.
|
||||
/// </summary>
|
||||
string Version,
|
||||
|
||||
/// <summary>
|
||||
/// New or updated projections.
|
||||
/// </summary>
|
||||
IReadOnlyList<ConsensusProjection> Added,
|
||||
|
||||
/// <summary>
|
||||
/// Removed projection keys.
|
||||
/// </summary>
|
||||
IReadOnlyList<ProjectionKey> Removed,
|
||||
|
||||
/// <summary>
|
||||
/// Incremental metadata.
|
||||
/// </summary>
|
||||
IncrementalMetadata Metadata);
|
||||
|
||||
/// <summary>
|
||||
/// Key identifying a projection.
|
||||
/// </summary>
|
||||
public sealed record ProjectionKey(
|
||||
string VulnerabilityId,
|
||||
string ProductKey,
|
||||
string? TenantId);
|
||||
|
||||
/// <summary>
|
||||
/// Metadata for incremental snapshot.
|
||||
/// </summary>
|
||||
public sealed record IncrementalMetadata(
|
||||
int AddedCount,
|
||||
int RemovedCount,
|
||||
DateTimeOffset? SinceTimestamp,
|
||||
string ContentHash);
|
||||
|
||||
/// <summary>
|
||||
/// Result of snapshot verification.
|
||||
/// </summary>
|
||||
public sealed record SnapshotVerificationResult(
|
||||
bool IsValid,
|
||||
string? ErrorMessage,
|
||||
int VerifiedCount,
|
||||
int MismatchCount,
|
||||
IReadOnlyList<VerificationMismatch>? Mismatches);
|
||||
|
||||
/// <summary>
|
||||
/// A mismatch found during verification.
|
||||
/// </summary>
|
||||
public sealed record VerificationMismatch(
|
||||
string VulnerabilityId,
|
||||
string ProductKey,
|
||||
string Field,
|
||||
string? ExpectedValue,
|
||||
string? ActualValue);
|
||||
|
||||
/// <summary>
|
||||
/// Export format.
|
||||
/// </summary>
|
||||
public enum ExportFormat
|
||||
{
|
||||
/// <summary>
|
||||
/// NDJSON (newline-delimited JSON).
|
||||
/// </summary>
|
||||
JsonLines,
|
||||
|
||||
/// <summary>
|
||||
/// Single JSON document.
|
||||
/// </summary>
|
||||
Json,
|
||||
|
||||
/// <summary>
|
||||
/// Compact binary format.
|
||||
/// </summary>
|
||||
Binary
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IConsensusExportService"/>.
|
||||
/// </summary>
|
||||
public sealed class ConsensusExportService : IConsensusExportService
|
||||
{
|
||||
private readonly IConsensusProjectionStore _projectionStore;
|
||||
|
||||
private const string SnapshotVersion = "1.0.0";
|
||||
|
||||
public ConsensusExportService(IConsensusProjectionStore projectionStore)
|
||||
{
|
||||
_projectionStore = projectionStore;
|
||||
}
|
||||
|
||||
public async Task<ConsensusSnapshot> CreateSnapshotAsync(
|
||||
SnapshotRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var query = new ProjectionQuery(
|
||||
TenantId: request.TenantId,
|
||||
VulnerabilityId: request.VulnerabilityIds?.FirstOrDefault(),
|
||||
ProductKey: request.ProductKeys?.FirstOrDefault(),
|
||||
Status: request.Status,
|
||||
Outcome: null,
|
||||
MinimumConfidence: request.MinimumConfidence,
|
||||
ComputedAfter: request.ComputedAfter,
|
||||
ComputedBefore: request.ComputedBefore,
|
||||
StatusChanged: null,
|
||||
Limit: request.MaxProjections ?? 10000,
|
||||
Offset: 0,
|
||||
SortBy: ProjectionSortField.ComputedAt,
|
||||
SortDescending: true);
|
||||
|
||||
var result = await _projectionStore.ListAsync(query, cancellationToken);
|
||||
|
||||
// Filter by additional criteria if needed
|
||||
var projections = result.Projections.ToList();
|
||||
|
||||
if (request.VulnerabilityIds is { Count: > 1 })
|
||||
{
|
||||
var vulnSet = new HashSet<string>(request.VulnerabilityIds);
|
||||
projections = projections.Where(p => vulnSet.Contains(p.VulnerabilityId)).ToList();
|
||||
}
|
||||
|
||||
if (request.ProductKeys is { Count: > 1 })
|
||||
{
|
||||
var productSet = new HashSet<string>(request.ProductKeys);
|
||||
projections = projections.Where(p => productSet.Contains(p.ProductKey)).ToList();
|
||||
}
|
||||
|
||||
// Load history if requested
|
||||
List<ConsensusProjection>? history = null;
|
||||
if (request.IncludeHistory)
|
||||
{
|
||||
history = [];
|
||||
foreach (var projection in projections.Take(100)) // Limit history loading
|
||||
{
|
||||
var projHistory = await _projectionStore.GetHistoryAsync(
|
||||
projection.VulnerabilityId,
|
||||
projection.ProductKey,
|
||||
projection.TenantId,
|
||||
10,
|
||||
cancellationToken);
|
||||
history.AddRange(projHistory);
|
||||
}
|
||||
}
|
||||
|
||||
var statusCounts = projections
|
||||
.GroupBy(p => p.Status)
|
||||
.ToDictionary(g => g.Key, g => g.Count());
|
||||
|
||||
var snapshotId = $"snap-{Guid.NewGuid():N}";
|
||||
var contentHash = ComputeContentHash(projections);
|
||||
|
||||
return new ConsensusSnapshot(
|
||||
SnapshotId: snapshotId,
|
||||
CreatedAt: DateTimeOffset.UtcNow,
|
||||
Version: SnapshotVersion,
|
||||
TenantId: request.TenantId,
|
||||
Projections: projections,
|
||||
History: history,
|
||||
Metadata: new SnapshotMetadata(
|
||||
TotalProjections: projections.Count,
|
||||
TotalHistoryEntries: history?.Count ?? 0,
|
||||
OldestProjection: projections.Min(p => (DateTimeOffset?)p.ComputedAt),
|
||||
NewestProjection: projections.Max(p => (DateTimeOffset?)p.ComputedAt),
|
||||
StatusCounts: statusCounts,
|
||||
ContentHash: contentHash,
|
||||
CreatedBy: "VexLens"));
|
||||
}
|
||||
|
||||
public async Task ExportToStreamAsync(
|
||||
ConsensusSnapshot snapshot,
|
||||
Stream outputStream,
|
||||
ExportFormat format = ExportFormat.JsonLines,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var options = new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = format == ExportFormat.Json,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
switch (format)
|
||||
{
|
||||
case ExportFormat.JsonLines:
|
||||
await ExportAsJsonLinesAsync(snapshot, outputStream, options, cancellationToken);
|
||||
break;
|
||||
|
||||
case ExportFormat.Json:
|
||||
await JsonSerializer.SerializeAsync(outputStream, snapshot, options, cancellationToken);
|
||||
break;
|
||||
|
||||
case ExportFormat.Binary:
|
||||
// For binary format, use JSON with no indentation as a simple binary-ish format
|
||||
options.WriteIndented = false;
|
||||
await JsonSerializer.SerializeAsync(outputStream, snapshot, options, cancellationToken);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IncrementalSnapshot> CreateIncrementalSnapshotAsync(
|
||||
string? lastSnapshotId,
|
||||
DateTimeOffset? since,
|
||||
SnapshotRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Get current projections
|
||||
var currentRequest = request with { ComputedAfter = since };
|
||||
var current = await CreateSnapshotAsync(currentRequest, cancellationToken);
|
||||
|
||||
// For a true incremental, we'd compare with the previous snapshot
|
||||
// Here we just return new/updated since the timestamp
|
||||
var snapshotId = $"snap-inc-{Guid.NewGuid():N}";
|
||||
var contentHash = ComputeContentHash(current.Projections);
|
||||
|
||||
return new IncrementalSnapshot(
|
||||
SnapshotId: snapshotId,
|
||||
PreviousSnapshotId: lastSnapshotId,
|
||||
CreatedAt: DateTimeOffset.UtcNow,
|
||||
Version: SnapshotVersion,
|
||||
Added: current.Projections,
|
||||
Removed: [], // Would need previous snapshot to determine removed
|
||||
Metadata: new IncrementalMetadata(
|
||||
AddedCount: current.Projections.Count,
|
||||
RemovedCount: 0,
|
||||
SinceTimestamp: since,
|
||||
ContentHash: contentHash));
|
||||
}
|
||||
|
||||
public async Task<SnapshotVerificationResult> VerifySnapshotAsync(
|
||||
ConsensusSnapshot snapshot,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var mismatches = new List<VerificationMismatch>();
|
||||
var verifiedCount = 0;
|
||||
|
||||
foreach (var projection in snapshot.Projections)
|
||||
{
|
||||
var current = await _projectionStore.GetLatestAsync(
|
||||
projection.VulnerabilityId,
|
||||
projection.ProductKey,
|
||||
projection.TenantId,
|
||||
cancellationToken);
|
||||
|
||||
if (current == null)
|
||||
{
|
||||
mismatches.Add(new VerificationMismatch(
|
||||
projection.VulnerabilityId,
|
||||
projection.ProductKey,
|
||||
"existence",
|
||||
"exists",
|
||||
"not found"));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check key fields
|
||||
if (current.Status != projection.Status)
|
||||
{
|
||||
mismatches.Add(new VerificationMismatch(
|
||||
projection.VulnerabilityId,
|
||||
projection.ProductKey,
|
||||
"status",
|
||||
projection.Status.ToString(),
|
||||
current.Status.ToString()));
|
||||
}
|
||||
|
||||
if (Math.Abs(current.ConfidenceScore - projection.ConfidenceScore) > 0.001)
|
||||
{
|
||||
mismatches.Add(new VerificationMismatch(
|
||||
projection.VulnerabilityId,
|
||||
projection.ProductKey,
|
||||
"confidenceScore",
|
||||
projection.ConfidenceScore.ToString("F4"),
|
||||
current.ConfidenceScore.ToString("F4")));
|
||||
}
|
||||
|
||||
verifiedCount++;
|
||||
}
|
||||
|
||||
return new SnapshotVerificationResult(
|
||||
IsValid: mismatches.Count == 0,
|
||||
ErrorMessage: mismatches.Count > 0 ? $"{mismatches.Count} mismatch(es) found" : null,
|
||||
VerifiedCount: verifiedCount,
|
||||
MismatchCount: mismatches.Count,
|
||||
Mismatches: mismatches.Count > 0 ? mismatches : null);
|
||||
}
|
||||
|
||||
private static async Task ExportAsJsonLinesAsync(
|
||||
ConsensusSnapshot snapshot,
|
||||
Stream outputStream,
|
||||
JsonSerializerOptions options,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using var writer = new StreamWriter(outputStream, leaveOpen: true);
|
||||
|
||||
// Write header line
|
||||
var header = new
|
||||
{
|
||||
type = "header",
|
||||
snapshotId = snapshot.SnapshotId,
|
||||
createdAt = snapshot.CreatedAt,
|
||||
version = snapshot.Version,
|
||||
metadata = snapshot.Metadata
|
||||
};
|
||||
await writer.WriteLineAsync(JsonSerializer.Serialize(header, options));
|
||||
|
||||
// Write each projection
|
||||
foreach (var projection in snapshot.Projections)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var line = new { type = "projection", data = projection };
|
||||
await writer.WriteLineAsync(JsonSerializer.Serialize(line, options));
|
||||
}
|
||||
|
||||
// Write history if present
|
||||
if (snapshot.History != null)
|
||||
{
|
||||
foreach (var historyEntry in snapshot.History)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var line = new { type = "history", data = historyEntry };
|
||||
await writer.WriteLineAsync(JsonSerializer.Serialize(line, options));
|
||||
}
|
||||
}
|
||||
|
||||
// Write footer
|
||||
var footer = new
|
||||
{
|
||||
type = "footer",
|
||||
totalProjections = snapshot.Projections.Count,
|
||||
totalHistory = snapshot.History?.Count ?? 0,
|
||||
contentHash = snapshot.Metadata.ContentHash
|
||||
};
|
||||
await writer.WriteLineAsync(JsonSerializer.Serialize(footer, options));
|
||||
}
|
||||
|
||||
private static string ComputeContentHash(IReadOnlyList<ConsensusProjection> projections)
|
||||
{
|
||||
var data = string.Join("|", projections
|
||||
.OrderBy(p => p.VulnerabilityId)
|
||||
.ThenBy(p => p.ProductKey)
|
||||
.Select(p => $"{p.VulnerabilityId}:{p.ProductKey}:{p.Status}:{p.ConfidenceScore:F4}"));
|
||||
|
||||
var hash = System.Security.Cryptography.SHA256.HashData(
|
||||
System.Text.Encoding.UTF8.GetBytes(data));
|
||||
return Convert.ToHexString(hash).ToLowerInvariant()[..32];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extensions for export configuration.
|
||||
/// </summary>
|
||||
public static class ConsensusExportExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a snapshot request for full export.
|
||||
/// </summary>
|
||||
public static SnapshotRequest FullExportRequest(string? tenantId = null)
|
||||
{
|
||||
return new SnapshotRequest(
|
||||
TenantId: tenantId,
|
||||
VulnerabilityIds: null,
|
||||
ProductKeys: null,
|
||||
MinimumConfidence: null,
|
||||
Status: null,
|
||||
ComputedAfter: null,
|
||||
ComputedBefore: null,
|
||||
IncludeHistory: false,
|
||||
MaxProjections: null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a snapshot request for mirror bundle export.
|
||||
/// </summary>
|
||||
public static SnapshotRequest MirrorBundleRequest(
|
||||
string? tenantId = null,
|
||||
double minimumConfidence = 0.5,
|
||||
bool includeHistory = false)
|
||||
{
|
||||
return new SnapshotRequest(
|
||||
TenantId: tenantId,
|
||||
VulnerabilityIds: null,
|
||||
ProductKeys: null,
|
||||
MinimumConfidence: minimumConfidence,
|
||||
Status: null,
|
||||
ComputedAfter: null,
|
||||
ComputedBefore: null,
|
||||
IncludeHistory: includeHistory,
|
||||
MaxProjections: 100000);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user