up
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,247 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Attestation.Dsse;
|
||||
using StellaOps.Excititor.Attestation.Signing;
|
||||
using StellaOps.Excititor.Core.Evidence;
|
||||
|
||||
namespace StellaOps.Excititor.Attestation.Evidence;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IVexEvidenceAttestor"/> that creates DSSE attestations for evidence manifests.
|
||||
/// </summary>
|
||||
public sealed class VexEvidenceAttestor : IVexEvidenceAttestor
|
||||
{
|
||||
internal const string PayloadType = "application/vnd.in-toto+json";
|
||||
|
||||
private readonly IVexSigner _signer;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<VexEvidenceAttestor> _logger;
|
||||
private readonly JsonSerializerOptions _serializerOptions;
|
||||
|
||||
public VexEvidenceAttestor(
|
||||
IVexSigner signer,
|
||||
ILogger<VexEvidenceAttestor> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_signer = signer ?? throw new ArgumentNullException(nameof(signer));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_serializerOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
WriteIndented = false,
|
||||
};
|
||||
_serializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase));
|
||||
}
|
||||
|
||||
public async ValueTask<VexEvidenceAttestationResult> AttestManifestAsync(
|
||||
VexLockerManifest manifest,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
|
||||
var attestedAt = _timeProvider.GetUtcNow();
|
||||
var attestationId = CreateAttestationId(manifest, attestedAt);
|
||||
|
||||
// Build in-toto statement
|
||||
var predicate = VexEvidenceAttestationPredicate.FromManifest(manifest);
|
||||
var subject = new VexEvidenceInTotoSubject(
|
||||
manifest.ManifestId,
|
||||
ImmutableDictionary<string, string>.Empty.Add("sha256", manifest.MerkleRoot.Replace("sha256:", "")));
|
||||
|
||||
var statement = new InTotoStatementDto
|
||||
{
|
||||
Type = VexEvidenceInTotoStatement.InTotoStatementType,
|
||||
PredicateType = VexEvidenceInTotoStatement.EvidenceLockerPredicateType,
|
||||
Subject = new[] { new InTotoSubjectDto { Name = subject.Name, Digest = subject.Digest } },
|
||||
Predicate = new InTotoPredicateDto
|
||||
{
|
||||
ManifestId = predicate.ManifestId,
|
||||
Tenant = predicate.Tenant,
|
||||
MerkleRoot = predicate.MerkleRoot,
|
||||
ItemCount = predicate.ItemCount,
|
||||
CreatedAt = predicate.CreatedAt,
|
||||
Metadata = predicate.Metadata.Count > 0 ? predicate.Metadata : null
|
||||
}
|
||||
};
|
||||
|
||||
// Serialize and sign
|
||||
var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(statement, _serializerOptions);
|
||||
var signatureResult = await _signer.SignAsync(payloadBytes, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Build DSSE envelope
|
||||
var envelope = new DsseEnvelope(
|
||||
Convert.ToBase64String(payloadBytes),
|
||||
PayloadType,
|
||||
new[] { new DsseSignature(signatureResult.Signature, signatureResult.KeyId) });
|
||||
|
||||
var envelopeJson = JsonSerializer.Serialize(envelope, _serializerOptions);
|
||||
var envelopeHash = ComputeHash(envelopeJson);
|
||||
|
||||
// Create signed manifest
|
||||
var signedManifest = manifest.WithSignature(signatureResult.Signature);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Evidence attestation created for manifest {ManifestId}: attestation={AttestationId} hash={Hash}",
|
||||
manifest.ManifestId,
|
||||
attestationId,
|
||||
envelopeHash);
|
||||
|
||||
return new VexEvidenceAttestationResult(
|
||||
signedManifest,
|
||||
envelopeJson,
|
||||
envelopeHash,
|
||||
attestationId,
|
||||
attestedAt);
|
||||
}
|
||||
|
||||
public ValueTask<VexEvidenceVerificationResult> VerifyAttestationAsync(
|
||||
VexLockerManifest manifest,
|
||||
string dsseEnvelopeJson,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
if (string.IsNullOrWhiteSpace(dsseEnvelopeJson))
|
||||
{
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure("DSSE envelope is required."));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var envelope = JsonSerializer.Deserialize<DsseEnvelope>(dsseEnvelopeJson, _serializerOptions);
|
||||
if (envelope is null)
|
||||
{
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure("Invalid DSSE envelope format."));
|
||||
}
|
||||
|
||||
// Decode payload and verify it matches the manifest
|
||||
var payloadBytes = Convert.FromBase64String(envelope.Payload);
|
||||
var statement = JsonSerializer.Deserialize<InTotoStatementDto>(payloadBytes, _serializerOptions);
|
||||
if (statement is null)
|
||||
{
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure("Invalid in-toto statement format."));
|
||||
}
|
||||
|
||||
// Verify statement type
|
||||
if (statement.Type != VexEvidenceInTotoStatement.InTotoStatementType)
|
||||
{
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
|
||||
$"Invalid statement type: expected {VexEvidenceInTotoStatement.InTotoStatementType}, got {statement.Type}"));
|
||||
}
|
||||
|
||||
// Verify predicate type
|
||||
if (statement.PredicateType != VexEvidenceInTotoStatement.EvidenceLockerPredicateType)
|
||||
{
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
|
||||
$"Invalid predicate type: expected {VexEvidenceInTotoStatement.EvidenceLockerPredicateType}, got {statement.PredicateType}"));
|
||||
}
|
||||
|
||||
// Verify manifest ID matches
|
||||
if (statement.Predicate?.ManifestId != manifest.ManifestId)
|
||||
{
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
|
||||
$"Manifest ID mismatch: expected {manifest.ManifestId}, got {statement.Predicate?.ManifestId}"));
|
||||
}
|
||||
|
||||
// Verify Merkle root matches
|
||||
if (statement.Predicate?.MerkleRoot != manifest.MerkleRoot)
|
||||
{
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
|
||||
$"Merkle root mismatch: expected {manifest.MerkleRoot}, got {statement.Predicate?.MerkleRoot}"));
|
||||
}
|
||||
|
||||
// Verify item count matches
|
||||
if (statement.Predicate?.ItemCount != manifest.Items.Length)
|
||||
{
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure(
|
||||
$"Item count mismatch: expected {manifest.Items.Length}, got {statement.Predicate?.ItemCount}"));
|
||||
}
|
||||
|
||||
var diagnostics = ImmutableDictionary.CreateBuilder<string, string>();
|
||||
diagnostics.Add("envelope_hash", ComputeHash(dsseEnvelopeJson));
|
||||
diagnostics.Add("verified_at", _timeProvider.GetUtcNow().ToString("O"));
|
||||
|
||||
_logger.LogDebug("Evidence attestation verified for manifest {ManifestId}", manifest.ManifestId);
|
||||
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Success(diagnostics.ToImmutable()));
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse DSSE envelope for manifest {ManifestId}", manifest.ManifestId);
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure($"JSON parse error: {ex.Message}"));
|
||||
}
|
||||
catch (FormatException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to decode base64 payload for manifest {ManifestId}", manifest.ManifestId);
|
||||
return ValueTask.FromResult(VexEvidenceVerificationResult.Failure($"Base64 decode error: {ex.Message}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static string CreateAttestationId(VexLockerManifest manifest, DateTimeOffset timestamp)
|
||||
{
|
||||
var normalized = manifest.Tenant.ToLowerInvariant();
|
||||
var date = timestamp.ToString("yyyyMMddHHmmssfff");
|
||||
return $"attest:evidence:{normalized}:{date}";
|
||||
}
|
||||
|
||||
private static string ComputeHash(string content)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(content);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
// DTOs for JSON serialization
|
||||
private sealed record InTotoStatementDto
|
||||
{
|
||||
[JsonPropertyName("_type")]
|
||||
public string? Type { get; init; }
|
||||
|
||||
[JsonPropertyName("predicateType")]
|
||||
public string? PredicateType { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public InTotoSubjectDto[]? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("predicate")]
|
||||
public InTotoPredicateDto? Predicate { get; init; }
|
||||
}
|
||||
|
||||
private sealed record InTotoSubjectDto
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; init; }
|
||||
|
||||
[JsonPropertyName("digest")]
|
||||
public ImmutableDictionary<string, string>? Digest { get; init; }
|
||||
}
|
||||
|
||||
private sealed record InTotoPredicateDto
|
||||
{
|
||||
[JsonPropertyName("manifestId")]
|
||||
public string? ManifestId { get; init; }
|
||||
|
||||
[JsonPropertyName("tenant")]
|
||||
public string? Tenant { get; init; }
|
||||
|
||||
[JsonPropertyName("merkleRoot")]
|
||||
public string? MerkleRoot { get; init; }
|
||||
|
||||
[JsonPropertyName("itemCount")]
|
||||
public int? ItemCount { get; init; }
|
||||
|
||||
[JsonPropertyName("createdAt")]
|
||||
public DateTimeOffset? CreatedAt { get; init; }
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
public ImmutableDictionary<string, string>? Metadata { get; init; }
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Excititor.Attestation.Dsse;
|
||||
using StellaOps.Excititor.Attestation.Evidence;
|
||||
using StellaOps.Excititor.Attestation.Transparency;
|
||||
using StellaOps.Excititor.Attestation.Verification;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Evidence;
|
||||
|
||||
namespace StellaOps.Excititor.Attestation.Extensions;
|
||||
|
||||
@@ -14,14 +16,15 @@ public static class VexAttestationServiceCollectionExtensions
|
||||
services.AddSingleton<VexAttestationMetrics>();
|
||||
services.AddSingleton<IVexAttestationVerifier, VexAttestationVerifier>();
|
||||
services.AddSingleton<IVexAttestationClient, VexAttestationClient>();
|
||||
services.AddSingleton<IVexEvidenceAttestor, VexEvidenceAttestor>();
|
||||
return services;
|
||||
}
|
||||
|
||||
public static IServiceCollection AddVexRekorClient(this IServiceCollection services, Action<RekorHttpClientOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
services.Configure(configure);
|
||||
services.AddHttpClient<ITransparencyLogClient, RekorHttpClient>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
public static IServiceCollection AddVexRekorClient(this IServiceCollection services, Action<RekorHttpClientOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
services.Configure(configure);
|
||||
services.AddHttpClient<ITransparencyLogClient, RekorHttpClient>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Canonicalization;
|
||||
|
||||
/// <summary>
|
||||
/// Canonicalizes advisory and vulnerability identifiers to a stable <see cref="VexCanonicalAdvisoryKey"/>.
|
||||
/// Preserves original identifiers in the Links collection for traceability.
|
||||
/// </summary>
|
||||
public sealed class VexAdvisoryKeyCanonicalizer
|
||||
{
|
||||
private static readonly Regex CvePattern = new(
|
||||
@"^CVE-\d{4}-\d{4,}$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex GhsaPattern = new(
|
||||
@"^GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex RhsaPattern = new(
|
||||
@"^RH[A-Z]{2}-\d{4}:\d+$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex DsaPattern = new(
|
||||
@"^DSA-\d+(-\d+)?$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex UsnPattern = new(
|
||||
@"^USN-\d+(-\d+)?$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex MsrcPattern = new(
|
||||
@"^(ADV|CVE)-\d{4}-\d+$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
/// <summary>
|
||||
/// Canonicalizes an advisory identifier and extracts scope metadata.
|
||||
/// </summary>
|
||||
/// <param name="originalId">The original advisory/vulnerability identifier.</param>
|
||||
/// <param name="aliases">Optional alias identifiers to include in links.</param>
|
||||
/// <returns>A canonical advisory key with preserved original links.</returns>
|
||||
public VexCanonicalAdvisoryKey Canonicalize(string originalId, IEnumerable<string>? aliases = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(originalId);
|
||||
|
||||
var normalized = originalId.Trim().ToUpperInvariant();
|
||||
var scope = DetermineScope(normalized);
|
||||
var canonicalKey = BuildCanonicalKey(normalized, scope);
|
||||
|
||||
var linksBuilder = ImmutableArray.CreateBuilder<VexAdvisoryLink>();
|
||||
|
||||
// Add the original identifier as a link
|
||||
linksBuilder.Add(new VexAdvisoryLink(
|
||||
originalId.Trim(),
|
||||
DetermineIdType(normalized),
|
||||
isOriginal: true));
|
||||
|
||||
// Add aliases as links
|
||||
if (aliases is not null)
|
||||
{
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { normalized };
|
||||
foreach (var alias in aliases)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(alias))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalizedAlias = alias.Trim();
|
||||
if (!seen.Add(normalizedAlias.ToUpperInvariant()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
linksBuilder.Add(new VexAdvisoryLink(
|
||||
normalizedAlias,
|
||||
DetermineIdType(normalizedAlias.ToUpperInvariant()),
|
||||
isOriginal: false));
|
||||
}
|
||||
}
|
||||
|
||||
return new VexCanonicalAdvisoryKey(
|
||||
canonicalKey,
|
||||
scope,
|
||||
linksBuilder.ToImmutable());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts CVE identifier from aliases if the original is not a CVE.
|
||||
/// </summary>
|
||||
public string? ExtractCveFromAliases(IEnumerable<string>? aliases)
|
||||
{
|
||||
if (aliases is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var alias in aliases)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(alias))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = alias.Trim().ToUpperInvariant();
|
||||
if (CvePattern.IsMatch(normalized))
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static VexAdvisoryScope DetermineScope(string normalizedId)
|
||||
{
|
||||
if (CvePattern.IsMatch(normalizedId))
|
||||
{
|
||||
return VexAdvisoryScope.Global;
|
||||
}
|
||||
|
||||
if (GhsaPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return VexAdvisoryScope.Ecosystem;
|
||||
}
|
||||
|
||||
if (RhsaPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return VexAdvisoryScope.Vendor;
|
||||
}
|
||||
|
||||
if (DsaPattern.IsMatch(normalizedId) || UsnPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return VexAdvisoryScope.Distribution;
|
||||
}
|
||||
|
||||
if (MsrcPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return VexAdvisoryScope.Vendor;
|
||||
}
|
||||
|
||||
return VexAdvisoryScope.Unknown;
|
||||
}
|
||||
|
||||
private static string BuildCanonicalKey(string normalizedId, VexAdvisoryScope scope)
|
||||
{
|
||||
// CVE is the most authoritative global identifier
|
||||
if (CvePattern.IsMatch(normalizedId))
|
||||
{
|
||||
return normalizedId;
|
||||
}
|
||||
|
||||
// For non-CVE identifiers, prefix with scope indicator for disambiguation
|
||||
var prefix = scope switch
|
||||
{
|
||||
VexAdvisoryScope.Ecosystem => "ECO",
|
||||
VexAdvisoryScope.Vendor => "VND",
|
||||
VexAdvisoryScope.Distribution => "DST",
|
||||
_ => "UNK",
|
||||
};
|
||||
|
||||
return $"{prefix}:{normalizedId}";
|
||||
}
|
||||
|
||||
private static string DetermineIdType(string normalizedId)
|
||||
{
|
||||
if (CvePattern.IsMatch(normalizedId))
|
||||
{
|
||||
return "cve";
|
||||
}
|
||||
|
||||
if (GhsaPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return "ghsa";
|
||||
}
|
||||
|
||||
if (RhsaPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return "rhsa";
|
||||
}
|
||||
|
||||
if (DsaPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return "dsa";
|
||||
}
|
||||
|
||||
if (UsnPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return "usn";
|
||||
}
|
||||
|
||||
if (MsrcPattern.IsMatch(normalizedId))
|
||||
{
|
||||
return "msrc";
|
||||
}
|
||||
|
||||
return "other";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a canonicalized advisory key with preserved original identifiers.
|
||||
/// </summary>
|
||||
public sealed record VexCanonicalAdvisoryKey
|
||||
{
|
||||
public VexCanonicalAdvisoryKey(
|
||||
string advisoryKey,
|
||||
VexAdvisoryScope scope,
|
||||
ImmutableArray<VexAdvisoryLink> links)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(advisoryKey))
|
||||
{
|
||||
throw new ArgumentException("Advisory key must be provided.", nameof(advisoryKey));
|
||||
}
|
||||
|
||||
AdvisoryKey = advisoryKey.Trim();
|
||||
Scope = scope;
|
||||
Links = links.IsDefault ? ImmutableArray<VexAdvisoryLink>.Empty : links;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The canonical advisory key used for correlation and storage.
|
||||
/// </summary>
|
||||
public string AdvisoryKey { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The scope/authority level of the advisory.
|
||||
/// </summary>
|
||||
public VexAdvisoryScope Scope { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Original and alias identifiers preserved for traceability.
|
||||
/// </summary>
|
||||
public ImmutableArray<VexAdvisoryLink> Links { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Returns the original identifier if available.
|
||||
/// </summary>
|
||||
public string? OriginalId => Links.FirstOrDefault(l => l.IsOriginal)?.Identifier;
|
||||
|
||||
/// <summary>
|
||||
/// Returns all non-original alias identifiers.
|
||||
/// </summary>
|
||||
public IEnumerable<string> Aliases => Links.Where(l => !l.IsOriginal).Select(l => l.Identifier);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a link to an original or alias advisory identifier.
|
||||
/// </summary>
|
||||
public sealed record VexAdvisoryLink
|
||||
{
|
||||
public VexAdvisoryLink(string identifier, string type, bool isOriginal)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(identifier))
|
||||
{
|
||||
throw new ArgumentException("Identifier must be provided.", nameof(identifier));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
throw new ArgumentException("Type must be provided.", nameof(type));
|
||||
}
|
||||
|
||||
Identifier = identifier.Trim();
|
||||
Type = type.Trim().ToLowerInvariant();
|
||||
IsOriginal = isOriginal;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The advisory identifier value.
|
||||
/// </summary>
|
||||
public string Identifier { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The type of identifier (cve, ghsa, rhsa, dsa, usn, msrc, other).
|
||||
/// </summary>
|
||||
public string Type { get; }
|
||||
|
||||
/// <summary>
|
||||
/// True if this is the original identifier provided at ingest time.
|
||||
/// </summary>
|
||||
public bool IsOriginal { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The scope/authority level of an advisory.
|
||||
/// </summary>
|
||||
public enum VexAdvisoryScope
|
||||
{
|
||||
/// <summary>
|
||||
/// Unknown or unclassified scope.
|
||||
/// </summary>
|
||||
Unknown = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Global identifiers (e.g., CVE).
|
||||
/// </summary>
|
||||
Global = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Ecosystem-specific identifiers (e.g., GHSA).
|
||||
/// </summary>
|
||||
Ecosystem = 2,
|
||||
|
||||
/// <summary>
|
||||
/// Vendor-specific identifiers (e.g., RHSA, MSRC).
|
||||
/// </summary>
|
||||
Vendor = 3,
|
||||
|
||||
/// <summary>
|
||||
/// Distribution-specific identifiers (e.g., DSA, USN).
|
||||
/// </summary>
|
||||
Distribution = 4,
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Canonicalization;
|
||||
|
||||
/// <summary>
|
||||
/// Canonicalizes product identifiers (PURL, CPE, OS package names) to a stable <see cref="VexCanonicalProductKey"/>.
|
||||
/// Preserves original identifiers in the Links collection for traceability.
|
||||
/// </summary>
|
||||
public sealed class VexProductKeyCanonicalizer
|
||||
{
|
||||
private static readonly Regex PurlPattern = new(
|
||||
@"^pkg:[a-z0-9]+/",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex CpePattern = new(
|
||||
@"^cpe:(2\.3:|/)",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
// RPM NEVRA format: name-[epoch:]version-release.arch
|
||||
// Release can contain dots (e.g., 1.el9), so we match until the last dot before arch
|
||||
private static readonly Regex RpmNevraPattern = new(
|
||||
@"^(?<name>[a-zA-Z0-9_+-]+)-(?<epoch>\d+:)?(?<version>[^-]+)-(?<release>.+)\.(?<arch>x86_64|i686|noarch|aarch64|s390x|ppc64le|armv7hl|src)$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
// Debian packages use underscores as separators: name_version_arch or name_version
|
||||
// Must have at least one underscore to be considered a Debian package
|
||||
private static readonly Regex DebianPackagePattern = new(
|
||||
@"^(?<name>[a-z0-9][a-z0-9.+-]+)_(?<version>[^_]+)(_(?<arch>[a-z0-9-]+))?$",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
|
||||
|
||||
/// <summary>
|
||||
/// Canonicalizes a product identifier and extracts scope metadata.
|
||||
/// </summary>
|
||||
/// <param name="originalKey">The original product key/identifier.</param>
|
||||
/// <param name="purl">Optional PURL for the product.</param>
|
||||
/// <param name="cpe">Optional CPE for the product.</param>
|
||||
/// <param name="componentIdentifiers">Optional additional component identifiers.</param>
|
||||
/// <returns>A canonical product key with preserved original links.</returns>
|
||||
public VexCanonicalProductKey Canonicalize(
|
||||
string originalKey,
|
||||
string? purl = null,
|
||||
string? cpe = null,
|
||||
IEnumerable<string>? componentIdentifiers = null)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(originalKey);
|
||||
|
||||
// Check component identifiers for PURL if not provided directly
|
||||
var effectivePurl = purl ?? ExtractPurlFromIdentifiers(componentIdentifiers);
|
||||
var effectiveCpe = cpe ?? ExtractCpeFromIdentifiers(componentIdentifiers);
|
||||
|
||||
var keyType = DetermineKeyType(originalKey.Trim());
|
||||
var scope = DetermineScope(originalKey.Trim(), effectivePurl, effectiveCpe);
|
||||
var canonicalKey = BuildCanonicalKey(originalKey.Trim(), effectivePurl, effectiveCpe, keyType);
|
||||
|
||||
var linksBuilder = ImmutableArray.CreateBuilder<VexProductLink>();
|
||||
|
||||
// Add the original key as a link
|
||||
linksBuilder.Add(new VexProductLink(
|
||||
originalKey.Trim(),
|
||||
keyType.ToString().ToLowerInvariant(),
|
||||
isOriginal: true));
|
||||
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { originalKey.Trim() };
|
||||
|
||||
// Add PURL if different from original
|
||||
if (!string.IsNullOrWhiteSpace(purl) && seen.Add(purl.Trim()))
|
||||
{
|
||||
linksBuilder.Add(new VexProductLink(
|
||||
purl.Trim(),
|
||||
"purl",
|
||||
isOriginal: false));
|
||||
}
|
||||
|
||||
// Add CPE if different from original
|
||||
if (!string.IsNullOrWhiteSpace(cpe) && seen.Add(cpe.Trim()))
|
||||
{
|
||||
linksBuilder.Add(new VexProductLink(
|
||||
cpe.Trim(),
|
||||
"cpe",
|
||||
isOriginal: false));
|
||||
}
|
||||
|
||||
// Add component identifiers
|
||||
if (componentIdentifiers is not null)
|
||||
{
|
||||
foreach (var identifier in componentIdentifiers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(identifier))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalizedId = identifier.Trim();
|
||||
if (!seen.Add(normalizedId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var idType = DetermineKeyType(normalizedId);
|
||||
linksBuilder.Add(new VexProductLink(
|
||||
normalizedId,
|
||||
idType.ToString().ToLowerInvariant(),
|
||||
isOriginal: false));
|
||||
}
|
||||
}
|
||||
|
||||
return new VexCanonicalProductKey(
|
||||
canonicalKey,
|
||||
scope,
|
||||
keyType,
|
||||
linksBuilder.ToImmutable());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts PURL from component identifiers if available.
|
||||
/// </summary>
|
||||
public string? ExtractPurlFromIdentifiers(IEnumerable<string>? identifiers)
|
||||
{
|
||||
if (identifiers is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var id in identifiers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (PurlPattern.IsMatch(id.Trim()))
|
||||
{
|
||||
return id.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts CPE from component identifiers if available.
|
||||
/// </summary>
|
||||
public string? ExtractCpeFromIdentifiers(IEnumerable<string>? identifiers)
|
||||
{
|
||||
if (identifiers is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var id in identifiers)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (CpePattern.IsMatch(id.Trim()))
|
||||
{
|
||||
return id.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static VexProductKeyType DetermineKeyType(string key)
|
||||
{
|
||||
if (PurlPattern.IsMatch(key))
|
||||
{
|
||||
return VexProductKeyType.Purl;
|
||||
}
|
||||
|
||||
if (CpePattern.IsMatch(key))
|
||||
{
|
||||
return VexProductKeyType.Cpe;
|
||||
}
|
||||
|
||||
if (RpmNevraPattern.IsMatch(key))
|
||||
{
|
||||
return VexProductKeyType.RpmNevra;
|
||||
}
|
||||
|
||||
if (DebianPackagePattern.IsMatch(key))
|
||||
{
|
||||
return VexProductKeyType.DebianPackage;
|
||||
}
|
||||
|
||||
if (key.StartsWith("oci:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return VexProductKeyType.OciImage;
|
||||
}
|
||||
|
||||
if (key.StartsWith("platform:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return VexProductKeyType.Platform;
|
||||
}
|
||||
|
||||
return VexProductKeyType.Other;
|
||||
}
|
||||
|
||||
private static VexProductScope DetermineScope(string key, string? purl, string? cpe)
|
||||
{
|
||||
// PURL is the most authoritative
|
||||
if (!string.IsNullOrWhiteSpace(purl) || PurlPattern.IsMatch(key))
|
||||
{
|
||||
return VexProductScope.Package;
|
||||
}
|
||||
|
||||
// CPE is next
|
||||
if (!string.IsNullOrWhiteSpace(cpe) || CpePattern.IsMatch(key))
|
||||
{
|
||||
return VexProductScope.Component;
|
||||
}
|
||||
|
||||
// OS packages
|
||||
if (RpmNevraPattern.IsMatch(key) || DebianPackagePattern.IsMatch(key))
|
||||
{
|
||||
return VexProductScope.OsPackage;
|
||||
}
|
||||
|
||||
// OCI images
|
||||
if (key.StartsWith("oci:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return VexProductScope.Container;
|
||||
}
|
||||
|
||||
// Platforms
|
||||
if (key.StartsWith("platform:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return VexProductScope.Platform;
|
||||
}
|
||||
|
||||
return VexProductScope.Unknown;
|
||||
}
|
||||
|
||||
private static string BuildCanonicalKey(string key, string? purl, string? cpe, VexProductKeyType keyType)
|
||||
{
|
||||
// Prefer PURL as canonical key
|
||||
if (!string.IsNullOrWhiteSpace(purl))
|
||||
{
|
||||
return NormalizePurl(purl.Trim());
|
||||
}
|
||||
|
||||
if (PurlPattern.IsMatch(key))
|
||||
{
|
||||
return NormalizePurl(key);
|
||||
}
|
||||
|
||||
// Fall back to CPE
|
||||
if (!string.IsNullOrWhiteSpace(cpe))
|
||||
{
|
||||
return NormalizeCpe(cpe.Trim());
|
||||
}
|
||||
|
||||
if (CpePattern.IsMatch(key))
|
||||
{
|
||||
return NormalizeCpe(key);
|
||||
}
|
||||
|
||||
// For types that already have their prefix, return as-is
|
||||
if (keyType == VexProductKeyType.OciImage && key.StartsWith("oci:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return key;
|
||||
}
|
||||
|
||||
if (keyType == VexProductKeyType.Platform && key.StartsWith("platform:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return key;
|
||||
}
|
||||
|
||||
// For other types, prefix for disambiguation
|
||||
var prefix = keyType switch
|
||||
{
|
||||
VexProductKeyType.RpmNevra => "rpm",
|
||||
VexProductKeyType.DebianPackage => "deb",
|
||||
VexProductKeyType.OciImage => "oci",
|
||||
VexProductKeyType.Platform => "platform",
|
||||
_ => "product",
|
||||
};
|
||||
|
||||
return $"{prefix}:{key}";
|
||||
}
|
||||
|
||||
private static string NormalizePurl(string purl)
|
||||
{
|
||||
// Ensure lowercase scheme
|
||||
if (purl.StartsWith("PKG:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "pkg:" + purl.Substring(4);
|
||||
}
|
||||
|
||||
return purl;
|
||||
}
|
||||
|
||||
private static string NormalizeCpe(string cpe)
|
||||
{
|
||||
// Ensure lowercase scheme
|
||||
if (cpe.StartsWith("CPE:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "cpe:" + cpe.Substring(4);
|
||||
}
|
||||
|
||||
return cpe;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a canonicalized product key with preserved original identifiers.
|
||||
/// </summary>
|
||||
public sealed record VexCanonicalProductKey
|
||||
{
|
||||
public VexCanonicalProductKey(
|
||||
string productKey,
|
||||
VexProductScope scope,
|
||||
VexProductKeyType keyType,
|
||||
ImmutableArray<VexProductLink> links)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(productKey))
|
||||
{
|
||||
throw new ArgumentException("Product key must be provided.", nameof(productKey));
|
||||
}
|
||||
|
||||
ProductKey = productKey.Trim();
|
||||
Scope = scope;
|
||||
KeyType = keyType;
|
||||
Links = links.IsDefault ? ImmutableArray<VexProductLink>.Empty : links;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The canonical product key used for correlation and storage.
|
||||
/// </summary>
|
||||
public string ProductKey { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The scope/authority level of the product identifier.
|
||||
/// </summary>
|
||||
public VexProductScope Scope { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The type of the canonical key.
|
||||
/// </summary>
|
||||
public VexProductKeyType KeyType { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Original and alias identifiers preserved for traceability.
|
||||
/// </summary>
|
||||
public ImmutableArray<VexProductLink> Links { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Returns the original identifier if available.
|
||||
/// </summary>
|
||||
public string? OriginalKey => Links.FirstOrDefault(l => l.IsOriginal)?.Identifier;
|
||||
|
||||
/// <summary>
|
||||
/// Returns the PURL link if available.
|
||||
/// </summary>
|
||||
public string? Purl => Links.FirstOrDefault(l => l.Type == "purl")?.Identifier;
|
||||
|
||||
/// <summary>
|
||||
/// Returns the CPE link if available.
|
||||
/// </summary>
|
||||
public string? Cpe => Links.FirstOrDefault(l => l.Type == "cpe")?.Identifier;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a link to an original or alias product identifier.
|
||||
/// </summary>
|
||||
public sealed record VexProductLink
|
||||
{
|
||||
public VexProductLink(string identifier, string type, bool isOriginal)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(identifier))
|
||||
{
|
||||
throw new ArgumentException("Identifier must be provided.", nameof(identifier));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(type))
|
||||
{
|
||||
throw new ArgumentException("Type must be provided.", nameof(type));
|
||||
}
|
||||
|
||||
Identifier = identifier.Trim();
|
||||
Type = type.Trim().ToLowerInvariant();
|
||||
IsOriginal = isOriginal;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The product identifier value.
|
||||
/// </summary>
|
||||
public string Identifier { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The type of identifier (purl, cpe, rpm, deb, oci, platform, other).
|
||||
/// </summary>
|
||||
public string Type { get; }
|
||||
|
||||
/// <summary>
|
||||
/// True if this is the original identifier provided at ingest time.
|
||||
/// </summary>
|
||||
public bool IsOriginal { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The scope/authority level of a product identifier.
|
||||
/// </summary>
|
||||
public enum VexProductScope
|
||||
{
|
||||
/// <summary>
|
||||
/// Unknown or unclassified scope.
|
||||
/// </summary>
|
||||
Unknown = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Package-level identifier (PURL).
|
||||
/// </summary>
|
||||
Package = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Component-level identifier (CPE).
|
||||
/// </summary>
|
||||
Component = 2,
|
||||
|
||||
/// <summary>
|
||||
/// OS package identifier (RPM, DEB).
|
||||
/// </summary>
|
||||
OsPackage = 3,
|
||||
|
||||
/// <summary>
|
||||
/// Container image identifier.
|
||||
/// </summary>
|
||||
Container = 4,
|
||||
|
||||
/// <summary>
|
||||
/// Platform-level identifier.
|
||||
/// </summary>
|
||||
Platform = 5,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The type of product key identifier.
|
||||
/// </summary>
|
||||
public enum VexProductKeyType
|
||||
{
|
||||
/// <summary>
|
||||
/// Other/unknown type.
|
||||
/// </summary>
|
||||
Other = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Package URL (PURL).
|
||||
/// </summary>
|
||||
Purl = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Common Platform Enumeration (CPE).
|
||||
/// </summary>
|
||||
Cpe = 2,
|
||||
|
||||
/// <summary>
|
||||
/// RPM NEVRA format.
|
||||
/// </summary>
|
||||
RpmNevra = 3,
|
||||
|
||||
/// <summary>
|
||||
/// Debian package format.
|
||||
/// </summary>
|
||||
DebianPackage = 4,
|
||||
|
||||
/// <summary>
|
||||
/// OCI image reference.
|
||||
/// </summary>
|
||||
OciImage = 5,
|
||||
|
||||
/// <summary>
|
||||
/// Platform identifier.
|
||||
/// </summary>
|
||||
Platform = 6,
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Evidence;
|
||||
|
||||
/// <summary>
|
||||
/// Service interface for creating and verifying DSSE attestations on evidence locker manifests.
|
||||
/// </summary>
|
||||
public interface IVexEvidenceAttestor
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a DSSE attestation for the given manifest and returns the signed manifest.
|
||||
/// </summary>
|
||||
ValueTask<VexEvidenceAttestationResult> AttestManifestAsync(
|
||||
VexLockerManifest manifest,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Verifies an attestation for the given manifest.
|
||||
/// </summary>
|
||||
ValueTask<VexEvidenceVerificationResult> VerifyAttestationAsync(
|
||||
VexLockerManifest manifest,
|
||||
string dsseEnvelopeJson,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of attesting an evidence manifest.
|
||||
/// </summary>
|
||||
public sealed record VexEvidenceAttestationResult
|
||||
{
|
||||
public VexEvidenceAttestationResult(
|
||||
VexLockerManifest signedManifest,
|
||||
string dsseEnvelopeJson,
|
||||
string dsseEnvelopeHash,
|
||||
string attestationId,
|
||||
DateTimeOffset attestedAt)
|
||||
{
|
||||
SignedManifest = signedManifest ?? throw new ArgumentNullException(nameof(signedManifest));
|
||||
DsseEnvelopeJson = EnsureNotNullOrWhiteSpace(dsseEnvelopeJson, nameof(dsseEnvelopeJson));
|
||||
DsseEnvelopeHash = EnsureNotNullOrWhiteSpace(dsseEnvelopeHash, nameof(dsseEnvelopeHash));
|
||||
AttestationId = EnsureNotNullOrWhiteSpace(attestationId, nameof(attestationId));
|
||||
AttestedAt = attestedAt;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The manifest with the attestation signature attached.
|
||||
/// </summary>
|
||||
public VexLockerManifest SignedManifest { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The DSSE envelope as JSON.
|
||||
/// </summary>
|
||||
public string DsseEnvelopeJson { get; }
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the DSSE envelope.
|
||||
/// </summary>
|
||||
public string DsseEnvelopeHash { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Unique identifier for this attestation.
|
||||
/// </summary>
|
||||
public string AttestationId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// When the attestation was created.
|
||||
/// </summary>
|
||||
public DateTimeOffset AttestedAt { get; }
|
||||
|
||||
private static string EnsureNotNullOrWhiteSpace(string value, string name)
|
||||
=> string.IsNullOrWhiteSpace(value) ? throw new ArgumentException($"{name} must be provided.", name) : value.Trim();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of verifying an evidence attestation.
|
||||
/// </summary>
|
||||
public sealed record VexEvidenceVerificationResult
|
||||
{
|
||||
public VexEvidenceVerificationResult(
|
||||
bool isValid,
|
||||
string? failureReason = null,
|
||||
ImmutableDictionary<string, string>? diagnostics = null)
|
||||
{
|
||||
IsValid = isValid;
|
||||
FailureReason = failureReason?.Trim();
|
||||
Diagnostics = diagnostics ?? ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the attestation is valid.
|
||||
/// </summary>
|
||||
public bool IsValid { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Reason for failure if not valid.
|
||||
/// </summary>
|
||||
public string? FailureReason { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional diagnostic information.
|
||||
/// </summary>
|
||||
public ImmutableDictionary<string, string> Diagnostics { get; }
|
||||
|
||||
public static VexEvidenceVerificationResult Success(ImmutableDictionary<string, string>? diagnostics = null)
|
||||
=> new(true, null, diagnostics);
|
||||
|
||||
public static VexEvidenceVerificationResult Failure(string reason, ImmutableDictionary<string, string>? diagnostics = null)
|
||||
=> new(false, reason, diagnostics);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// in-toto statement for evidence locker attestations.
|
||||
/// </summary>
|
||||
public sealed record VexEvidenceInTotoStatement
|
||||
{
|
||||
public const string InTotoStatementType = "https://in-toto.io/Statement/v1";
|
||||
public const string EvidenceLockerPredicateType = "https://stella-ops.org/attestations/evidence-locker/v1";
|
||||
|
||||
public VexEvidenceInTotoStatement(
|
||||
ImmutableArray<VexEvidenceInTotoSubject> subjects,
|
||||
VexEvidenceAttestationPredicate predicate)
|
||||
{
|
||||
Type = InTotoStatementType;
|
||||
Subjects = subjects;
|
||||
PredicateType = EvidenceLockerPredicateType;
|
||||
Predicate = predicate ?? throw new ArgumentNullException(nameof(predicate));
|
||||
}
|
||||
|
||||
public string Type { get; }
|
||||
public ImmutableArray<VexEvidenceInTotoSubject> Subjects { get; }
|
||||
public string PredicateType { get; }
|
||||
public VexEvidenceAttestationPredicate Predicate { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Subject of an evidence locker attestation.
|
||||
/// </summary>
|
||||
public sealed record VexEvidenceInTotoSubject(
|
||||
string Name,
|
||||
ImmutableDictionary<string, string> Digest);
|
||||
|
||||
/// <summary>
|
||||
/// Predicate for evidence locker attestations.
|
||||
/// </summary>
|
||||
public sealed record VexEvidenceAttestationPredicate
|
||||
{
|
||||
public VexEvidenceAttestationPredicate(
|
||||
string manifestId,
|
||||
string tenant,
|
||||
string merkleRoot,
|
||||
int itemCount,
|
||||
DateTimeOffset createdAt,
|
||||
ImmutableDictionary<string, string>? metadata = null)
|
||||
{
|
||||
ManifestId = EnsureNotNullOrWhiteSpace(manifestId, nameof(manifestId));
|
||||
Tenant = EnsureNotNullOrWhiteSpace(tenant, nameof(tenant));
|
||||
MerkleRoot = EnsureNotNullOrWhiteSpace(merkleRoot, nameof(merkleRoot));
|
||||
ItemCount = itemCount;
|
||||
CreatedAt = createdAt;
|
||||
Metadata = metadata ?? ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
public string ManifestId { get; }
|
||||
public string Tenant { get; }
|
||||
public string MerkleRoot { get; }
|
||||
public int ItemCount { get; }
|
||||
public DateTimeOffset CreatedAt { get; }
|
||||
public ImmutableDictionary<string, string> Metadata { get; }
|
||||
|
||||
public static VexEvidenceAttestationPredicate FromManifest(VexLockerManifest manifest)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
return new VexEvidenceAttestationPredicate(
|
||||
manifest.ManifestId,
|
||||
manifest.Tenant,
|
||||
manifest.MerkleRoot,
|
||||
manifest.Items.Length,
|
||||
manifest.CreatedAt,
|
||||
manifest.Metadata);
|
||||
}
|
||||
|
||||
private static string EnsureNotNullOrWhiteSpace(string value, string name)
|
||||
=> string.IsNullOrWhiteSpace(value) ? throw new ArgumentException($"{name} must be provided.", name) : value.Trim();
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Evidence;
|
||||
|
||||
/// <summary>
|
||||
/// Service interface for building evidence locker payloads and Merkle manifests.
|
||||
/// </summary>
|
||||
public interface IVexEvidenceLockerService
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates an evidence snapshot item from an observation.
|
||||
/// </summary>
|
||||
VexEvidenceSnapshotItem CreateSnapshotItem(
|
||||
VexObservation observation,
|
||||
string linksetId,
|
||||
VexEvidenceProvenance? provenance = null);
|
||||
|
||||
/// <summary>
|
||||
/// Builds a locker manifest from a collection of observations.
|
||||
/// </summary>
|
||||
VexLockerManifest BuildManifest(
|
||||
string tenant,
|
||||
IEnumerable<VexObservation> observations,
|
||||
Func<VexObservation, string> linksetIdSelector,
|
||||
DateTimeOffset? timestamp = null,
|
||||
int sequence = 1,
|
||||
bool isSealed = false);
|
||||
|
||||
/// <summary>
|
||||
/// Builds a locker manifest from pre-built snapshot items.
|
||||
/// </summary>
|
||||
VexLockerManifest BuildManifest(
|
||||
string tenant,
|
||||
IEnumerable<VexEvidenceSnapshotItem> items,
|
||||
DateTimeOffset? timestamp = null,
|
||||
int sequence = 1,
|
||||
bool isSealed = false);
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a manifest's Merkle root against its items.
|
||||
/// </summary>
|
||||
bool VerifyManifest(VexLockerManifest manifest);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IVexEvidenceLockerService"/>.
|
||||
/// </summary>
|
||||
public sealed class VexEvidenceLockerService : IVexEvidenceLockerService
|
||||
{
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public VexEvidenceLockerService(TimeProvider? timeProvider = null)
|
||||
{
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public VexEvidenceSnapshotItem CreateSnapshotItem(
|
||||
VexObservation observation,
|
||||
string linksetId,
|
||||
VexEvidenceProvenance? provenance = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(observation);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(linksetId))
|
||||
{
|
||||
throw new ArgumentException("linksetId must be provided.", nameof(linksetId));
|
||||
}
|
||||
|
||||
return new VexEvidenceSnapshotItem(
|
||||
observationId: observation.ObservationId,
|
||||
providerId: observation.ProviderId,
|
||||
contentHash: observation.Upstream.ContentHash,
|
||||
linksetId: linksetId,
|
||||
dsseEnvelopeHash: null, // Populated by OBS-54-001
|
||||
provenance: provenance ?? VexEvidenceProvenance.Empty);
|
||||
}
|
||||
|
||||
public VexLockerManifest BuildManifest(
|
||||
string tenant,
|
||||
IEnumerable<VexObservation> observations,
|
||||
Func<VexObservation, string> linksetIdSelector,
|
||||
DateTimeOffset? timestamp = null,
|
||||
int sequence = 1,
|
||||
bool isSealed = false)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(observations);
|
||||
ArgumentNullException.ThrowIfNull(linksetIdSelector);
|
||||
|
||||
var items = observations
|
||||
.Where(o => o is not null)
|
||||
.Select(o => CreateSnapshotItem(o, linksetIdSelector(o)))
|
||||
.ToList();
|
||||
|
||||
return BuildManifest(tenant, items, timestamp, sequence, isSealed);
|
||||
}
|
||||
|
||||
public VexLockerManifest BuildManifest(
|
||||
string tenant,
|
||||
IEnumerable<VexEvidenceSnapshotItem> items,
|
||||
DateTimeOffset? timestamp = null,
|
||||
int sequence = 1,
|
||||
bool isSealed = false)
|
||||
{
|
||||
var ts = timestamp ?? _timeProvider.GetUtcNow();
|
||||
var manifestId = VexLockerManifest.CreateManifestId(tenant, ts, sequence);
|
||||
|
||||
var metadata = isSealed
|
||||
? System.Collections.Immutable.ImmutableDictionary<string, string>.Empty.Add("sealed", "true")
|
||||
: System.Collections.Immutable.ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
return new VexLockerManifest(
|
||||
tenant: tenant,
|
||||
manifestId: manifestId,
|
||||
createdAt: ts,
|
||||
items: items,
|
||||
signature: null,
|
||||
metadata: metadata);
|
||||
}
|
||||
|
||||
public bool VerifyManifest(VexLockerManifest manifest)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
|
||||
var expectedRoot = VexLockerManifest.ComputeMerkleRoot(manifest.Items);
|
||||
return string.Equals(manifest.MerkleRoot, expectedRoot, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Evidence;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a single evidence item in a locker payload for sealed-mode auditing.
|
||||
/// </summary>
|
||||
public sealed record VexEvidenceSnapshotItem
|
||||
{
|
||||
public VexEvidenceSnapshotItem(
|
||||
string observationId,
|
||||
string providerId,
|
||||
string contentHash,
|
||||
string linksetId,
|
||||
string? dsseEnvelopeHash = null,
|
||||
VexEvidenceProvenance? provenance = null)
|
||||
{
|
||||
ObservationId = EnsureNotNullOrWhiteSpace(observationId, nameof(observationId));
|
||||
ProviderId = EnsureNotNullOrWhiteSpace(providerId, nameof(providerId)).ToLowerInvariant();
|
||||
ContentHash = EnsureNotNullOrWhiteSpace(contentHash, nameof(contentHash));
|
||||
LinksetId = EnsureNotNullOrWhiteSpace(linksetId, nameof(linksetId));
|
||||
DsseEnvelopeHash = TrimToNull(dsseEnvelopeHash);
|
||||
Provenance = provenance ?? VexEvidenceProvenance.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The observation ID this evidence corresponds to.
|
||||
/// </summary>
|
||||
[JsonPropertyName("observationId")]
|
||||
public string ObservationId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The provider that supplied this evidence.
|
||||
/// </summary>
|
||||
[JsonPropertyName("providerId")]
|
||||
public string ProviderId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the raw observation content.
|
||||
/// </summary>
|
||||
[JsonPropertyName("contentHash")]
|
||||
public string ContentHash { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The linkset ID this evidence relates to.
|
||||
/// </summary>
|
||||
[JsonPropertyName("linksetId")]
|
||||
public string LinksetId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional DSSE envelope hash when attestations are enabled.
|
||||
/// </summary>
|
||||
[JsonPropertyName("dsseEnvelopeHash")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? DsseEnvelopeHash { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Provenance information for this evidence.
|
||||
/// </summary>
|
||||
[JsonPropertyName("provenance")]
|
||||
public VexEvidenceProvenance Provenance { get; }
|
||||
|
||||
private static string EnsureNotNullOrWhiteSpace(string value, string name)
|
||||
=> string.IsNullOrWhiteSpace(value) ? throw new ArgumentException($"{name} must be provided.", name) : value.Trim();
|
||||
|
||||
private static string? TrimToNull(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provenance information for evidence items.
|
||||
/// </summary>
|
||||
public sealed record VexEvidenceProvenance
|
||||
{
|
||||
public static readonly VexEvidenceProvenance Empty = new("ingest", null, null);
|
||||
|
||||
public VexEvidenceProvenance(
|
||||
string source,
|
||||
int? mirrorGeneration = null,
|
||||
string? exportCenterManifest = null)
|
||||
{
|
||||
Source = EnsureNotNullOrWhiteSpace(source, nameof(source)).ToLowerInvariant();
|
||||
MirrorGeneration = mirrorGeneration;
|
||||
ExportCenterManifest = TrimToNull(exportCenterManifest);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Source type: "mirror" or "ingest".
|
||||
/// </summary>
|
||||
[JsonPropertyName("source")]
|
||||
public string Source { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Mirror generation number when source is "mirror".
|
||||
/// </summary>
|
||||
[JsonPropertyName("mirrorGeneration")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public int? MirrorGeneration { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Export center manifest hash when available.
|
||||
/// </summary>
|
||||
[JsonPropertyName("exportCenterManifest")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? ExportCenterManifest { get; }
|
||||
|
||||
private static string EnsureNotNullOrWhiteSpace(string value, string name)
|
||||
=> string.IsNullOrWhiteSpace(value) ? throw new ArgumentException($"{name} must be provided.", name) : value.Trim();
|
||||
|
||||
private static string? TrimToNull(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Locker manifest containing evidence snapshots with Merkle root for verification.
|
||||
/// </summary>
|
||||
public sealed record VexLockerManifest
|
||||
{
|
||||
public VexLockerManifest(
|
||||
string tenant,
|
||||
string manifestId,
|
||||
DateTimeOffset createdAt,
|
||||
IEnumerable<VexEvidenceSnapshotItem> items,
|
||||
string? signature = null,
|
||||
ImmutableDictionary<string, string>? metadata = null)
|
||||
{
|
||||
Tenant = EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant();
|
||||
ManifestId = EnsureNotNullOrWhiteSpace(manifestId, nameof(manifestId));
|
||||
CreatedAt = createdAt.ToUniversalTime();
|
||||
Items = NormalizeItems(items);
|
||||
MerkleRoot = ComputeMerkleRoot(Items);
|
||||
Signature = TrimToNull(signature);
|
||||
Metadata = NormalizeMetadata(metadata);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tenant this manifest belongs to.
|
||||
/// </summary>
|
||||
[JsonPropertyName("tenant")]
|
||||
public string Tenant { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Unique manifest identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("manifestId")]
|
||||
public string ManifestId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// When this manifest was created.
|
||||
/// </summary>
|
||||
[JsonPropertyName("createdAt")]
|
||||
public DateTimeOffset CreatedAt { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Evidence items in deterministic order.
|
||||
/// </summary>
|
||||
[JsonPropertyName("items")]
|
||||
public ImmutableArray<VexEvidenceSnapshotItem> Items { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Merkle root computed over item content hashes.
|
||||
/// </summary>
|
||||
[JsonPropertyName("merkleRoot")]
|
||||
public string MerkleRoot { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional DSSE signature (populated by OBS-54-001).
|
||||
/// </summary>
|
||||
[JsonPropertyName("signature")]
|
||||
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
public string? Signature { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional metadata (e.g., sealed mode flag).
|
||||
/// </summary>
|
||||
[JsonPropertyName("metadata")]
|
||||
public ImmutableDictionary<string, string> Metadata { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new manifest with an attached signature.
|
||||
/// </summary>
|
||||
public VexLockerManifest WithSignature(string signature)
|
||||
{
|
||||
return new VexLockerManifest(
|
||||
Tenant,
|
||||
ManifestId,
|
||||
CreatedAt,
|
||||
Items,
|
||||
signature,
|
||||
Metadata);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a deterministic manifest ID.
|
||||
/// </summary>
|
||||
public static string CreateManifestId(string tenant, DateTimeOffset timestamp, int sequence)
|
||||
{
|
||||
var normalizedTenant = (tenant ?? "default").Trim().ToLowerInvariant();
|
||||
var date = timestamp.ToUniversalTime().ToString("yyyy-MM-dd");
|
||||
return $"locker:excititor:{normalizedTenant}:{date}:{sequence:D4}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes Merkle root from a list of hashes.
|
||||
/// </summary>
|
||||
public static string ComputeMerkleRoot(ImmutableArray<VexEvidenceSnapshotItem> items)
|
||||
{
|
||||
if (items.Length == 0)
|
||||
{
|
||||
return "sha256:" + Convert.ToHexString(SHA256.HashData(Array.Empty<byte>())).ToLowerInvariant();
|
||||
}
|
||||
|
||||
var hashes = items
|
||||
.Select(i => i.ContentHash.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)
|
||||
? i.ContentHash[7..]
|
||||
: i.ContentHash)
|
||||
.ToList();
|
||||
|
||||
return ComputeMerkleRootFromHashes(hashes);
|
||||
}
|
||||
|
||||
private static string ComputeMerkleRootFromHashes(List<string> hashes)
|
||||
{
|
||||
if (hashes.Count == 0)
|
||||
{
|
||||
return "sha256:" + Convert.ToHexString(SHA256.HashData(Array.Empty<byte>())).ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (hashes.Count == 1)
|
||||
{
|
||||
return "sha256:" + hashes[0].ToLowerInvariant();
|
||||
}
|
||||
|
||||
// Pad to even number if necessary
|
||||
if (hashes.Count % 2 != 0)
|
||||
{
|
||||
hashes.Add(hashes[^1]);
|
||||
}
|
||||
|
||||
var nextLevel = new List<string>();
|
||||
for (var i = 0; i < hashes.Count; i += 2)
|
||||
{
|
||||
var combined = hashes[i].ToLowerInvariant() + hashes[i + 1].ToLowerInvariant();
|
||||
var bytes = Convert.FromHexString(combined);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
nextLevel.Add(Convert.ToHexString(hash).ToLowerInvariant());
|
||||
}
|
||||
|
||||
return ComputeMerkleRootFromHashes(nextLevel);
|
||||
}
|
||||
|
||||
private static ImmutableArray<VexEvidenceSnapshotItem> NormalizeItems(IEnumerable<VexEvidenceSnapshotItem>? items)
|
||||
{
|
||||
if (items is null)
|
||||
{
|
||||
return ImmutableArray<VexEvidenceSnapshotItem>.Empty;
|
||||
}
|
||||
|
||||
// Sort by observationId, then providerId for deterministic ordering
|
||||
return items
|
||||
.Where(i => i is not null)
|
||||
.OrderBy(i => i.ObservationId, StringComparer.Ordinal)
|
||||
.ThenBy(i => i.ProviderId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableDictionary<string, string> NormalizeMetadata(ImmutableDictionary<string, string>? metadata)
|
||||
{
|
||||
if (metadata is null || metadata.Count == 0)
|
||||
{
|
||||
return ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
|
||||
foreach (var pair in metadata.OrderBy(kv => kv.Key, StringComparer.Ordinal))
|
||||
{
|
||||
var key = TrimToNull(pair.Key);
|
||||
var value = TrimToNull(pair.Value);
|
||||
if (key is null || value is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
builder[key] = value;
|
||||
}
|
||||
|
||||
return builder.ToImmutable();
|
||||
}
|
||||
|
||||
private static string EnsureNotNullOrWhiteSpace(string value, string name)
|
||||
=> string.IsNullOrWhiteSpace(value) ? throw new ArgumentException($"{name} must be provided.", name) : value.Trim();
|
||||
|
||||
private static string? TrimToNull(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Publishes vex.linkset.updated events to downstream consumers.
|
||||
/// Implementations may persist to MongoDB, publish to NATS, or both.
|
||||
/// </summary>
|
||||
public interface IVexLinksetEventPublisher
|
||||
{
|
||||
/// <summary>
|
||||
/// Publishes a linkset updated event.
|
||||
/// </summary>
|
||||
Task PublishAsync(VexLinksetUpdatedEvent @event, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Publishes multiple linkset updated events in a batch.
|
||||
/// </summary>
|
||||
Task PublishManyAsync(IEnumerable<VexLinksetUpdatedEvent> events, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Persistence abstraction for VEX linksets with tenant-isolated operations.
|
||||
/// Linksets correlate observations and capture conflict annotations.
|
||||
/// </summary>
|
||||
public interface IVexLinksetStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Persists a new linkset. Returns true if inserted, false if it already exists.
|
||||
/// </summary>
|
||||
ValueTask<bool> InsertAsync(
|
||||
VexLinkset linkset,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Persists or updates a linkset. Returns true if inserted, false if updated.
|
||||
/// </summary>
|
||||
ValueTask<bool> UpsertAsync(
|
||||
VexLinkset linkset,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a linkset by tenant and linkset ID.
|
||||
/// </summary>
|
||||
ValueTask<VexLinkset?> GetByIdAsync(
|
||||
string tenant,
|
||||
string linksetId,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves or creates a linkset for the given vulnerability and product key.
|
||||
/// </summary>
|
||||
ValueTask<VexLinkset> GetOrCreateAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Finds linksets by vulnerability ID.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<VexLinkset>> FindByVulnerabilityAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Finds linksets by product key.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<VexLinkset>> FindByProductKeyAsync(
|
||||
string tenant,
|
||||
string productKey,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Finds linksets that have disagreements (conflicts).
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<VexLinkset>> FindWithConflictsAsync(
|
||||
string tenant,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Finds linksets by provider ID.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<VexLinkset>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a linkset by tenant and linkset ID. Returns true if deleted.
|
||||
/// </summary>
|
||||
ValueTask<bool> DeleteAsync(
|
||||
string tenant,
|
||||
string linksetId,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the count of linksets for the specified tenant.
|
||||
/// </summary>
|
||||
ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the count of linksets with conflicts for the specified tenant.
|
||||
/// </summary>
|
||||
ValueTask<long> CountWithConflictsAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Persistence abstraction for VEX observations with tenant-isolated write operations.
|
||||
/// </summary>
|
||||
public interface IVexObservationStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Persists a new observation. Returns true if inserted, false if it already exists.
|
||||
/// </summary>
|
||||
ValueTask<bool> InsertAsync(
|
||||
VexObservation observation,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Persists or updates an observation. Returns true if inserted, false if updated.
|
||||
/// </summary>
|
||||
ValueTask<bool> UpsertAsync(
|
||||
VexObservation observation,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Persists multiple observations in a batch. Returns the count of newly inserted observations.
|
||||
/// </summary>
|
||||
ValueTask<int> InsertManyAsync(
|
||||
string tenant,
|
||||
IEnumerable<VexObservation> observations,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves an observation by tenant and observation ID.
|
||||
/// </summary>
|
||||
ValueTask<VexObservation?> GetByIdAsync(
|
||||
string tenant,
|
||||
string observationId,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves observations for a specific vulnerability and product key.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<VexObservation>> FindByVulnerabilityAndProductAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves observations by provider.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<VexObservation>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes an observation by tenant and observation ID. Returns true if deleted.
|
||||
/// </summary>
|
||||
ValueTask<bool> DeleteAsync(
|
||||
string tenant,
|
||||
string observationId,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the count of observations for the specified tenant.
|
||||
/// </summary>
|
||||
ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Service interface for emitting timeline events during ingest/linkset operations.
|
||||
/// Implementations should emit events asynchronously without blocking the main operation.
|
||||
/// </summary>
|
||||
public interface IVexTimelineEventEmitter
|
||||
{
|
||||
/// <summary>
|
||||
/// Emits a timeline event for an observation ingest operation.
|
||||
/// </summary>
|
||||
ValueTask EmitObservationIngestAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
string streamId,
|
||||
string traceId,
|
||||
string observationId,
|
||||
string evidenceHash,
|
||||
string justificationSummary,
|
||||
ImmutableDictionary<string, string>? attributes = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Emits a timeline event for a linkset update operation.
|
||||
/// </summary>
|
||||
ValueTask EmitLinksetUpdateAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
string streamId,
|
||||
string traceId,
|
||||
string linksetId,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
string payloadHash,
|
||||
string justificationSummary,
|
||||
ImmutableDictionary<string, string>? attributes = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Emits a timeline event for a generic operation.
|
||||
/// </summary>
|
||||
ValueTask EmitAsync(
|
||||
TimelineEvent evt,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Emits multiple timeline events in a batch.
|
||||
/// </summary>
|
||||
ValueTask EmitBatchAsync(
|
||||
string tenant,
|
||||
IEnumerable<TimelineEvent> events,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Well-known timeline event types for Excititor operations.
|
||||
/// </summary>
|
||||
public static class VexTimelineEventTypes
|
||||
{
|
||||
/// <summary>
|
||||
/// An observation was ingested.
|
||||
/// </summary>
|
||||
public const string ObservationIngested = "vex.observation.ingested";
|
||||
|
||||
/// <summary>
|
||||
/// An observation was updated.
|
||||
/// </summary>
|
||||
public const string ObservationUpdated = "vex.observation.updated";
|
||||
|
||||
/// <summary>
|
||||
/// An observation was superseded by another.
|
||||
/// </summary>
|
||||
public const string ObservationSuperseded = "vex.observation.superseded";
|
||||
|
||||
/// <summary>
|
||||
/// A linkset was created.
|
||||
/// </summary>
|
||||
public const string LinksetCreated = "vex.linkset.created";
|
||||
|
||||
/// <summary>
|
||||
/// A linkset was updated with new observations.
|
||||
/// </summary>
|
||||
public const string LinksetUpdated = "vex.linkset.updated";
|
||||
|
||||
/// <summary>
|
||||
/// A linkset conflict was detected.
|
||||
/// </summary>
|
||||
public const string LinksetConflictDetected = "vex.linkset.conflict_detected";
|
||||
|
||||
/// <summary>
|
||||
/// A linkset conflict was resolved.
|
||||
/// </summary>
|
||||
public const string LinksetConflictResolved = "vex.linkset.conflict_resolved";
|
||||
|
||||
/// <summary>
|
||||
/// Evidence was sealed to the locker.
|
||||
/// </summary>
|
||||
public const string EvidenceSealed = "vex.evidence.sealed";
|
||||
|
||||
/// <summary>
|
||||
/// An attestation was attached.
|
||||
/// </summary>
|
||||
public const string AttestationAttached = "vex.attestation.attached";
|
||||
|
||||
/// <summary>
|
||||
/// An attestation was verified.
|
||||
/// </summary>
|
||||
public const string AttestationVerified = "vex.attestation.verified";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Well-known attribute keys for timeline events.
|
||||
/// </summary>
|
||||
public static class VexTimelineEventAttributes
|
||||
{
|
||||
public const string ObservationId = "observation_id";
|
||||
public const string LinksetId = "linkset_id";
|
||||
public const string VulnerabilityId = "vulnerability_id";
|
||||
public const string ProductKey = "product_key";
|
||||
public const string Status = "status";
|
||||
public const string ConflictType = "conflict_type";
|
||||
public const string AttestationId = "attestation_id";
|
||||
public const string SupersededBy = "superseded_by";
|
||||
public const string Supersedes = "supersedes";
|
||||
public const string ObservationCount = "observation_count";
|
||||
public const string ConflictCount = "conflict_count";
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Persistence abstraction for VEX timeline events.
|
||||
/// Timeline events capture ingest/linkset changes with trace IDs, justification summaries,
|
||||
/// and evidence hashes so downstream systems can replay raw facts chronologically.
|
||||
/// </summary>
|
||||
public interface IVexTimelineEventStore
|
||||
{
|
||||
/// <summary>
|
||||
/// Persists a new timeline event. Returns the event ID if successful.
|
||||
/// </summary>
|
||||
ValueTask<string> InsertAsync(
|
||||
TimelineEvent evt,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Persists multiple timeline events in a batch. Returns the count of successfully inserted events.
|
||||
/// </summary>
|
||||
ValueTask<int> InsertManyAsync(
|
||||
string tenant,
|
||||
IEnumerable<TimelineEvent> events,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves timeline events for a tenant within a time range.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<TimelineEvent>> FindByTimeRangeAsync(
|
||||
string tenant,
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves timeline events by trace ID for correlation.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<TimelineEvent>> FindByTraceIdAsync(
|
||||
string tenant,
|
||||
string traceId,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves timeline events by provider ID.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<TimelineEvent>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves timeline events by event type.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<TimelineEvent>> FindByEventTypeAsync(
|
||||
string tenant,
|
||||
string eventType,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves the most recent timeline events for a tenant.
|
||||
/// </summary>
|
||||
ValueTask<IReadOnlyList<TimelineEvent>> GetRecentAsync(
|
||||
string tenant,
|
||||
int limit,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves a single timeline event by ID.
|
||||
/// </summary>
|
||||
ValueTask<TimelineEvent?> GetByIdAsync(
|
||||
string tenant,
|
||||
string eventId,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the count of timeline events for the specified tenant.
|
||||
/// </summary>
|
||||
ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the count of timeline events for the specified tenant within a time range.
|
||||
/// </summary>
|
||||
ValueTask<long> CountInRangeAsync(
|
||||
string tenant,
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a VEX linkset correlating multiple observations for a specific
|
||||
/// vulnerability and product key. Linksets capture disagreements (conflicts)
|
||||
/// between providers without deciding a winner.
|
||||
/// </summary>
|
||||
public sealed record VexLinkset
|
||||
{
|
||||
public VexLinkset(
|
||||
string linksetId,
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
IEnumerable<VexLinksetObservationRefModel> observations,
|
||||
IEnumerable<VexObservationDisagreement>? disagreements = null,
|
||||
DateTimeOffset? createdAt = null,
|
||||
DateTimeOffset? updatedAt = null)
|
||||
{
|
||||
LinksetId = VexObservation.EnsureNotNullOrWhiteSpace(linksetId, nameof(linksetId));
|
||||
Tenant = VexObservation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant();
|
||||
VulnerabilityId = VexObservation.EnsureNotNullOrWhiteSpace(vulnerabilityId, nameof(vulnerabilityId));
|
||||
ProductKey = VexObservation.EnsureNotNullOrWhiteSpace(productKey, nameof(productKey));
|
||||
Observations = NormalizeObservations(observations);
|
||||
Disagreements = NormalizeDisagreements(disagreements);
|
||||
CreatedAt = (createdAt ?? DateTimeOffset.UtcNow).ToUniversalTime();
|
||||
UpdatedAt = (updatedAt ?? CreatedAt).ToUniversalTime();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unique identifier for this linkset. Typically a SHA256 hash over
|
||||
/// (tenant, vulnerabilityId, productKey) for deterministic addressing.
|
||||
/// </summary>
|
||||
public string LinksetId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Tenant identifier (normalized to lowercase).
|
||||
/// </summary>
|
||||
public string Tenant { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The vulnerability identifier (CVE, GHSA, vendor ID).
|
||||
/// </summary>
|
||||
public string VulnerabilityId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Product key (typically a PURL or CPE).
|
||||
/// </summary>
|
||||
public string ProductKey { get; }
|
||||
|
||||
/// <summary>
|
||||
/// References to observations that contribute to this linkset.
|
||||
/// </summary>
|
||||
public ImmutableArray<VexLinksetObservationRefModel> Observations { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Conflict annotations capturing disagreements between providers.
|
||||
/// </summary>
|
||||
public ImmutableArray<VexObservationDisagreement> Disagreements { get; }
|
||||
|
||||
/// <summary>
|
||||
/// When this linkset was first created.
|
||||
/// </summary>
|
||||
public DateTimeOffset CreatedAt { get; }
|
||||
|
||||
/// <summary>
|
||||
/// When this linkset was last updated.
|
||||
/// </summary>
|
||||
public DateTimeOffset UpdatedAt { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Distinct provider IDs contributing to this linkset.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> ProviderIds =>
|
||||
Observations.Select(o => o.ProviderId)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(p => p, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Distinct statuses observed in this linkset.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> Statuses =>
|
||||
Observations.Select(o => o.Status)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
/// <summary>
|
||||
/// Whether this linkset contains disagreements (conflicts).
|
||||
/// </summary>
|
||||
public bool HasConflicts => !Disagreements.IsDefaultOrEmpty && Disagreements.Length > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Confidence level based on the linkset state.
|
||||
/// </summary>
|
||||
public VexLinksetConfidence Confidence
|
||||
{
|
||||
get
|
||||
{
|
||||
if (HasConflicts)
|
||||
{
|
||||
return VexLinksetConfidence.Low;
|
||||
}
|
||||
|
||||
if (Observations.Length == 0)
|
||||
{
|
||||
return VexLinksetConfidence.Low;
|
||||
}
|
||||
|
||||
var distinctStatuses = Statuses.Count;
|
||||
if (distinctStatuses > 1)
|
||||
{
|
||||
return VexLinksetConfidence.Low;
|
||||
}
|
||||
|
||||
var distinctProviders = ProviderIds.Count;
|
||||
if (distinctProviders >= 2)
|
||||
{
|
||||
return VexLinksetConfidence.High;
|
||||
}
|
||||
|
||||
return VexLinksetConfidence.Medium;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a deterministic linkset ID from key components.
|
||||
/// </summary>
|
||||
public static string CreateLinksetId(string tenant, string vulnerabilityId, string productKey)
|
||||
{
|
||||
var normalizedTenant = (tenant ?? string.Empty).Trim().ToLowerInvariant();
|
||||
var normalizedVuln = (vulnerabilityId ?? string.Empty).Trim();
|
||||
var normalizedProduct = (productKey ?? string.Empty).Trim();
|
||||
|
||||
var input = $"{normalizedTenant}|{normalizedVuln}|{normalizedProduct}";
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input));
|
||||
return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new linkset with updated observations and recomputed disagreements.
|
||||
/// </summary>
|
||||
public VexLinkset WithObservations(
|
||||
IEnumerable<VexLinksetObservationRefModel> observations,
|
||||
IEnumerable<VexObservationDisagreement>? disagreements = null)
|
||||
{
|
||||
return new VexLinkset(
|
||||
LinksetId,
|
||||
Tenant,
|
||||
VulnerabilityId,
|
||||
ProductKey,
|
||||
observations,
|
||||
disagreements,
|
||||
CreatedAt,
|
||||
DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
private static ImmutableArray<VexLinksetObservationRefModel> NormalizeObservations(
|
||||
IEnumerable<VexLinksetObservationRefModel>? observations)
|
||||
{
|
||||
if (observations is null)
|
||||
{
|
||||
return ImmutableArray<VexLinksetObservationRefModel>.Empty;
|
||||
}
|
||||
|
||||
var set = new SortedSet<VexLinksetObservationRefModel>(VexLinksetObservationRefComparer.Instance);
|
||||
foreach (var item in observations)
|
||||
{
|
||||
if (item is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var obsId = VexObservation.TrimToNull(item.ObservationId);
|
||||
var provider = VexObservation.TrimToNull(item.ProviderId);
|
||||
var status = VexObservation.TrimToNull(item.Status);
|
||||
if (obsId is null || provider is null || status is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
double? clamped = item.Confidence is null ? null : Math.Clamp(item.Confidence.Value, 0.0, 1.0);
|
||||
set.Add(new VexLinksetObservationRefModel(obsId, provider, status, clamped));
|
||||
}
|
||||
|
||||
return set.Count == 0 ? ImmutableArray<VexLinksetObservationRefModel>.Empty : set.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static ImmutableArray<VexObservationDisagreement> NormalizeDisagreements(
|
||||
IEnumerable<VexObservationDisagreement>? disagreements)
|
||||
{
|
||||
if (disagreements is null)
|
||||
{
|
||||
return ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
}
|
||||
|
||||
var set = new SortedSet<VexObservationDisagreement>(DisagreementComparer.Instance);
|
||||
foreach (var disagreement in disagreements)
|
||||
{
|
||||
if (disagreement is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalizedProvider = VexObservation.TrimToNull(disagreement.ProviderId);
|
||||
var normalizedStatus = VexObservation.TrimToNull(disagreement.Status);
|
||||
|
||||
if (normalizedProvider is null || normalizedStatus is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalizedJustification = VexObservation.TrimToNull(disagreement.Justification);
|
||||
double? clampedConfidence = disagreement.Confidence is null
|
||||
? null
|
||||
: Math.Clamp(disagreement.Confidence.Value, 0.0, 1.0);
|
||||
|
||||
set.Add(new VexObservationDisagreement(
|
||||
normalizedProvider,
|
||||
normalizedStatus,
|
||||
normalizedJustification,
|
||||
clampedConfidence));
|
||||
}
|
||||
|
||||
return set.Count == 0 ? ImmutableArray<VexObservationDisagreement>.Empty : set.ToImmutableArray();
|
||||
}
|
||||
|
||||
private sealed class DisagreementComparer : IComparer<VexObservationDisagreement>
|
||||
{
|
||||
public static readonly DisagreementComparer Instance = new();
|
||||
|
||||
public int Compare(VexObservationDisagreement? x, VexObservationDisagreement? y)
|
||||
{
|
||||
if (ReferenceEquals(x, y))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (x is null)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (y is null)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
var providerCompare = StringComparer.OrdinalIgnoreCase.Compare(x.ProviderId, y.ProviderId);
|
||||
if (providerCompare != 0)
|
||||
{
|
||||
return providerCompare;
|
||||
}
|
||||
|
||||
var statusCompare = StringComparer.OrdinalIgnoreCase.Compare(x.Status, y.Status);
|
||||
if (statusCompare != 0)
|
||||
{
|
||||
return statusCompare;
|
||||
}
|
||||
|
||||
var justificationCompare = StringComparer.OrdinalIgnoreCase.Compare(
|
||||
x.Justification ?? string.Empty,
|
||||
y.Justification ?? string.Empty);
|
||||
if (justificationCompare != 0)
|
||||
{
|
||||
return justificationCompare;
|
||||
}
|
||||
|
||||
return Nullable.Compare(x.Confidence, y.Confidence);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Confidence level for a linkset based on agreement between providers.
|
||||
/// </summary>
|
||||
public enum VexLinksetConfidence
|
||||
{
|
||||
/// <summary>
|
||||
/// Low confidence: conflicts exist or insufficient observations.
|
||||
/// </summary>
|
||||
Low,
|
||||
|
||||
/// <summary>
|
||||
/// Medium confidence: single provider or consistent observations.
|
||||
/// </summary>
|
||||
Medium,
|
||||
|
||||
/// <summary>
|
||||
/// High confidence: multiple providers agree.
|
||||
/// </summary>
|
||||
High
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Observations;
|
||||
|
||||
/// <summary>
|
||||
/// Computes disagreements (conflicts) from VEX observations without choosing winners.
|
||||
/// Excititor remains aggregation-only; downstream consumers use disagreements to highlight
|
||||
/// conflicts and apply their own decision rules (AOC-19-002).
|
||||
/// </summary>
|
||||
public sealed class VexLinksetDisagreementService
|
||||
{
|
||||
/// <summary>
|
||||
/// Analyzes observations and returns disagreements where providers report different
|
||||
/// statuses or justifications for the same vulnerability/product combination.
|
||||
/// </summary>
|
||||
public ImmutableArray<VexObservationDisagreement> ComputeDisagreements(
|
||||
IEnumerable<VexObservation> observations)
|
||||
{
|
||||
if (observations is null)
|
||||
{
|
||||
return ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
}
|
||||
|
||||
var observationList = observations
|
||||
.Where(o => o is not null)
|
||||
.ToList();
|
||||
|
||||
if (observationList.Count < 2)
|
||||
{
|
||||
return ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
}
|
||||
|
||||
// Group by (vulnerabilityId, productKey)
|
||||
var groups = observationList
|
||||
.SelectMany(obs => obs.Statements.Select(stmt => (obs, stmt)))
|
||||
.GroupBy(x => new
|
||||
{
|
||||
VulnerabilityId = Normalize(x.stmt.VulnerabilityId),
|
||||
ProductKey = Normalize(x.stmt.ProductKey)
|
||||
});
|
||||
|
||||
var disagreements = new List<VexObservationDisagreement>();
|
||||
|
||||
foreach (var group in groups)
|
||||
{
|
||||
var groupDisagreements = DetectGroupDisagreements(group.ToList());
|
||||
disagreements.AddRange(groupDisagreements);
|
||||
}
|
||||
|
||||
return disagreements
|
||||
.Distinct(DisagreementComparer.Instance)
|
||||
.OrderBy(d => d.ProviderId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(d => d.Status, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Analyzes observations for a specific linkset and returns disagreements.
|
||||
/// </summary>
|
||||
public ImmutableArray<VexObservationDisagreement> ComputeDisagreementsForLinkset(
|
||||
IEnumerable<VexLinksetObservationRefModel> observationRefs)
|
||||
{
|
||||
if (observationRefs is null)
|
||||
{
|
||||
return ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
}
|
||||
|
||||
var refList = observationRefs
|
||||
.Where(r => r is not null)
|
||||
.ToList();
|
||||
|
||||
if (refList.Count < 2)
|
||||
{
|
||||
return ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
}
|
||||
|
||||
// Group by status to detect conflicts
|
||||
var statusGroups = refList
|
||||
.GroupBy(r => Normalize(r.Status))
|
||||
.ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (statusGroups.Count <= 1)
|
||||
{
|
||||
// All providers agree on status
|
||||
return ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
}
|
||||
|
||||
// Multiple statuses = disagreement
|
||||
// Generate disagreement entries for each provider-status combination
|
||||
var disagreements = refList
|
||||
.Select(r => new VexObservationDisagreement(
|
||||
providerId: r.ProviderId,
|
||||
status: r.Status,
|
||||
justification: null,
|
||||
confidence: ComputeConfidence(r.Status, statusGroups)))
|
||||
.Distinct(DisagreementComparer.Instance)
|
||||
.OrderBy(d => d.ProviderId, StringComparer.OrdinalIgnoreCase)
|
||||
.ThenBy(d => d.Status, StringComparer.OrdinalIgnoreCase)
|
||||
.ToImmutableArray();
|
||||
|
||||
return disagreements;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates a linkset with computed disagreements based on its observations.
|
||||
/// Returns a new linkset with updated disagreements.
|
||||
/// </summary>
|
||||
public VexLinkset UpdateLinksetDisagreements(VexLinkset linkset)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(linkset);
|
||||
|
||||
var disagreements = ComputeDisagreementsForLinkset(linkset.Observations);
|
||||
|
||||
return linkset.WithObservations(
|
||||
linkset.Observations,
|
||||
disagreements);
|
||||
}
|
||||
|
||||
private static IEnumerable<VexObservationDisagreement> DetectGroupDisagreements(
|
||||
List<(VexObservation obs, VexObservationStatement stmt)> group)
|
||||
{
|
||||
if (group.Count < 2)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Group by provider to get unique provider perspectives
|
||||
var byProvider = group
|
||||
.GroupBy(x => Normalize(x.obs.ProviderId))
|
||||
.Select(g => new
|
||||
{
|
||||
ProviderId = g.Key,
|
||||
Status = Normalize(g.First().stmt.Status.ToString()),
|
||||
Justification = g.First().stmt.Justification?.ToString()
|
||||
})
|
||||
.ToList();
|
||||
|
||||
// Count status frequencies
|
||||
var statusCounts = byProvider
|
||||
.GroupBy(p => p.Status, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(g => g.Key, g => g.Count(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// If all providers agree on status, no disagreement
|
||||
if (statusCounts.Count <= 1)
|
||||
{
|
||||
yield break;
|
||||
}
|
||||
|
||||
// Multiple statuses = disagreement
|
||||
// Report each provider's position as a disagreement
|
||||
var totalProviders = byProvider.Count;
|
||||
|
||||
foreach (var provider in byProvider)
|
||||
{
|
||||
var statusCount = statusCounts[provider.Status];
|
||||
var confidence = (double)statusCount / totalProviders;
|
||||
|
||||
yield return new VexObservationDisagreement(
|
||||
providerId: provider.ProviderId,
|
||||
status: provider.Status,
|
||||
justification: provider.Justification,
|
||||
confidence: confidence);
|
||||
}
|
||||
}
|
||||
|
||||
private static double ComputeConfidence(
|
||||
string status,
|
||||
Dictionary<string, List<VexLinksetObservationRefModel>> statusGroups)
|
||||
{
|
||||
var totalCount = statusGroups.Values.Sum(g => g.Count);
|
||||
if (totalCount == 0)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
if (statusGroups.TryGetValue(status, out var group))
|
||||
{
|
||||
return (double)group.Count / totalCount;
|
||||
}
|
||||
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
private static string Normalize(string value)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(value)
|
||||
? string.Empty
|
||||
: value.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private sealed class DisagreementComparer : IEqualityComparer<VexObservationDisagreement>
|
||||
{
|
||||
public static readonly DisagreementComparer Instance = new();
|
||||
|
||||
public bool Equals(VexObservationDisagreement? x, VexObservationDisagreement? y)
|
||||
{
|
||||
if (ReferenceEquals(x, y))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (x is null || y is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return string.Equals(x.ProviderId, y.ProviderId, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Status, y.Status, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(x.Justification, y.Justification, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public int GetHashCode(VexObservationDisagreement obj)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(obj.ProviderId, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Status, StringComparer.OrdinalIgnoreCase);
|
||||
hash.Add(obj.Justification, StringComparer.OrdinalIgnoreCase);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
using System;
|
||||
using System.Collections.Immutable;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StellaOps.Excititor.Core.Orchestration;
|
||||
|
||||
/// <summary>
|
||||
/// Client interface for the orchestrator worker SDK.
|
||||
/// Emits heartbeats, progress, and artifact hashes for deterministic, restartable ingestion.
|
||||
/// </summary>
|
||||
public interface IVexWorkerOrchestratorClient
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a new job context for a provider run.
|
||||
/// </summary>
|
||||
ValueTask<VexWorkerJobContext> StartJobAsync(
|
||||
string tenant,
|
||||
string connectorId,
|
||||
string? checkpoint,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Emits a heartbeat for the given job.
|
||||
/// </summary>
|
||||
ValueTask SendHeartbeatAsync(
|
||||
VexWorkerJobContext context,
|
||||
VexWorkerHeartbeat heartbeat,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Records an artifact produced during the job.
|
||||
/// </summary>
|
||||
ValueTask RecordArtifactAsync(
|
||||
VexWorkerJobContext context,
|
||||
VexWorkerArtifact artifact,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Marks the job as completed successfully.
|
||||
/// </summary>
|
||||
ValueTask CompleteJobAsync(
|
||||
VexWorkerJobContext context,
|
||||
VexWorkerJobResult result,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Marks the job as failed.
|
||||
/// </summary>
|
||||
ValueTask FailJobAsync(
|
||||
VexWorkerJobContext context,
|
||||
string errorCode,
|
||||
string? errorMessage,
|
||||
int? retryAfterSeconds,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Marks the job as failed with a classified error.
|
||||
/// </summary>
|
||||
ValueTask FailJobAsync(
|
||||
VexWorkerJobContext context,
|
||||
VexWorkerError error,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Polls for pending commands from the orchestrator.
|
||||
/// </summary>
|
||||
ValueTask<VexWorkerCommand?> GetPendingCommandAsync(
|
||||
VexWorkerJobContext context,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Acknowledges that a command has been processed.
|
||||
/// </summary>
|
||||
ValueTask AcknowledgeCommandAsync(
|
||||
VexWorkerJobContext context,
|
||||
long commandSequence,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Saves a checkpoint for resumable ingestion.
|
||||
/// </summary>
|
||||
ValueTask SaveCheckpointAsync(
|
||||
VexWorkerJobContext context,
|
||||
VexWorkerCheckpoint checkpoint,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Loads the most recent checkpoint for a connector.
|
||||
/// </summary>
|
||||
ValueTask<VexWorkerCheckpoint?> LoadCheckpointAsync(
|
||||
string connectorId,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Context for an active worker job.
|
||||
/// </summary>
|
||||
public sealed record VexWorkerJobContext
|
||||
{
|
||||
public VexWorkerJobContext(
|
||||
string tenant,
|
||||
string connectorId,
|
||||
Guid runId,
|
||||
string? checkpoint,
|
||||
DateTimeOffset startedAt)
|
||||
{
|
||||
Tenant = EnsureNotNullOrWhiteSpace(tenant, nameof(tenant));
|
||||
ConnectorId = EnsureNotNullOrWhiteSpace(connectorId, nameof(connectorId));
|
||||
RunId = runId;
|
||||
Checkpoint = checkpoint?.Trim();
|
||||
StartedAt = startedAt;
|
||||
}
|
||||
|
||||
public string Tenant { get; }
|
||||
public string ConnectorId { get; }
|
||||
public Guid RunId { get; }
|
||||
public string? Checkpoint { get; }
|
||||
public DateTimeOffset StartedAt { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Current sequence number for heartbeats.
|
||||
/// </summary>
|
||||
public long Sequence { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Increments and returns the next sequence number.
|
||||
/// </summary>
|
||||
public long NextSequence() => ++Sequence;
|
||||
|
||||
private static string EnsureNotNullOrWhiteSpace(string value, string name)
|
||||
=> string.IsNullOrWhiteSpace(value) ? throw new ArgumentException($"{name} must be provided.", name) : value.Trim();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Heartbeat status for orchestrator reporting.
|
||||
/// </summary>
|
||||
public enum VexWorkerHeartbeatStatus
|
||||
{
|
||||
Starting,
|
||||
Running,
|
||||
Paused,
|
||||
Throttled,
|
||||
Backfill,
|
||||
Failed,
|
||||
Succeeded
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Heartbeat payload for orchestrator.
|
||||
/// </summary>
|
||||
public sealed record VexWorkerHeartbeat(
|
||||
VexWorkerHeartbeatStatus Status,
|
||||
int? Progress,
|
||||
int? QueueDepth,
|
||||
string? LastArtifactHash,
|
||||
string? LastArtifactKind,
|
||||
string? ErrorCode,
|
||||
int? RetryAfterSeconds);
|
||||
|
||||
/// <summary>
|
||||
/// Artifact produced during ingestion.
|
||||
/// </summary>
|
||||
public sealed record VexWorkerArtifact(
|
||||
string Hash,
|
||||
string Kind,
|
||||
string? ProviderId,
|
||||
string? DocumentId,
|
||||
DateTimeOffset CreatedAt,
|
||||
ImmutableDictionary<string, string>? Metadata = null);
|
||||
|
||||
/// <summary>
|
||||
/// Result of a completed worker job.
|
||||
/// </summary>
|
||||
public sealed record VexWorkerJobResult(
|
||||
int DocumentsProcessed,
|
||||
int ClaimsGenerated,
|
||||
string? LastCheckpoint,
|
||||
string? LastArtifactHash,
|
||||
DateTimeOffset CompletedAt,
|
||||
ImmutableDictionary<string, string>? Metadata = null);
|
||||
|
||||
/// <summary>
|
||||
/// Commands issued by the orchestrator to control worker behavior.
|
||||
/// </summary>
|
||||
public enum VexWorkerCommandKind
|
||||
{
|
||||
/// <summary>
|
||||
/// Continue normal processing.
|
||||
/// </summary>
|
||||
Continue,
|
||||
|
||||
/// <summary>
|
||||
/// Pause processing until resumed.
|
||||
/// </summary>
|
||||
Pause,
|
||||
|
||||
/// <summary>
|
||||
/// Resume after a pause.
|
||||
/// </summary>
|
||||
Resume,
|
||||
|
||||
/// <summary>
|
||||
/// Apply throttling constraints.
|
||||
/// </summary>
|
||||
Throttle,
|
||||
|
||||
/// <summary>
|
||||
/// Retry the current operation.
|
||||
/// </summary>
|
||||
Retry,
|
||||
|
||||
/// <summary>
|
||||
/// Abort the current job.
|
||||
/// </summary>
|
||||
Abort
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Command received from the orchestrator.
|
||||
/// </summary>
|
||||
public sealed record VexWorkerCommand(
|
||||
VexWorkerCommandKind Kind,
|
||||
long Sequence,
|
||||
DateTimeOffset IssuedAt,
|
||||
DateTimeOffset? ExpiresAt,
|
||||
VexWorkerThrottleParams? Throttle,
|
||||
string? Reason);
|
||||
|
||||
/// <summary>
|
||||
/// Throttle parameters issued with a throttle command.
|
||||
/// </summary>
|
||||
public sealed record VexWorkerThrottleParams(
|
||||
int? RequestsPerMinute,
|
||||
int? BurstLimit,
|
||||
int? CooldownSeconds);
|
||||
|
||||
/// <summary>
|
||||
/// Classification of errors for orchestrator reporting.
|
||||
/// </summary>
|
||||
public enum VexWorkerErrorCategory
|
||||
{
|
||||
/// <summary>
|
||||
/// Unknown or unclassified error.
|
||||
/// </summary>
|
||||
Unknown,
|
||||
|
||||
/// <summary>
|
||||
/// Transient network or connectivity issues.
|
||||
/// </summary>
|
||||
Network,
|
||||
|
||||
/// <summary>
|
||||
/// Authentication or authorization failure.
|
||||
/// </summary>
|
||||
Authorization,
|
||||
|
||||
/// <summary>
|
||||
/// Rate limiting or throttling by upstream.
|
||||
/// </summary>
|
||||
RateLimited,
|
||||
|
||||
/// <summary>
|
||||
/// Invalid or malformed data from upstream.
|
||||
/// </summary>
|
||||
DataFormat,
|
||||
|
||||
/// <summary>
|
||||
/// Upstream service unavailable.
|
||||
/// </summary>
|
||||
ServiceUnavailable,
|
||||
|
||||
/// <summary>
|
||||
/// Internal processing error.
|
||||
/// </summary>
|
||||
Internal,
|
||||
|
||||
/// <summary>
|
||||
/// Configuration or setup error.
|
||||
/// </summary>
|
||||
Configuration,
|
||||
|
||||
/// <summary>
|
||||
/// Operation cancelled.
|
||||
/// </summary>
|
||||
Cancelled,
|
||||
|
||||
/// <summary>
|
||||
/// Operation timed out.
|
||||
/// </summary>
|
||||
Timeout
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Classified error for orchestrator reporting.
|
||||
/// </summary>
|
||||
public sealed record VexWorkerError
|
||||
{
|
||||
public VexWorkerError(
|
||||
string code,
|
||||
VexWorkerErrorCategory category,
|
||||
string message,
|
||||
bool retryable,
|
||||
int? retryAfterSeconds = null,
|
||||
string? stage = null,
|
||||
ImmutableDictionary<string, string>? details = null)
|
||||
{
|
||||
Code = code ?? throw new ArgumentNullException(nameof(code));
|
||||
Category = category;
|
||||
Message = message ?? string.Empty;
|
||||
Retryable = retryable;
|
||||
RetryAfterSeconds = retryAfterSeconds;
|
||||
Stage = stage;
|
||||
Details = details ?? ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
public string Code { get; }
|
||||
public VexWorkerErrorCategory Category { get; }
|
||||
public string Message { get; }
|
||||
public bool Retryable { get; }
|
||||
public int? RetryAfterSeconds { get; }
|
||||
public string? Stage { get; }
|
||||
public ImmutableDictionary<string, string> Details { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a transient network error.
|
||||
/// </summary>
|
||||
public static VexWorkerError Network(string message, int? retryAfterSeconds = 30)
|
||||
=> new("NETWORK_ERROR", VexWorkerErrorCategory.Network, message, retryable: true, retryAfterSeconds);
|
||||
|
||||
/// <summary>
|
||||
/// Creates an authorization error.
|
||||
/// </summary>
|
||||
public static VexWorkerError Authorization(string message)
|
||||
=> new("AUTH_ERROR", VexWorkerErrorCategory.Authorization, message, retryable: false);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a rate-limited error.
|
||||
/// </summary>
|
||||
public static VexWorkerError RateLimited(string message, int retryAfterSeconds)
|
||||
=> new("RATE_LIMITED", VexWorkerErrorCategory.RateLimited, message, retryable: true, retryAfterSeconds);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a service unavailable error.
|
||||
/// </summary>
|
||||
public static VexWorkerError ServiceUnavailable(string message, int? retryAfterSeconds = 60)
|
||||
=> new("SERVICE_UNAVAILABLE", VexWorkerErrorCategory.ServiceUnavailable, message, retryable: true, retryAfterSeconds);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a data format error.
|
||||
/// </summary>
|
||||
public static VexWorkerError DataFormat(string message)
|
||||
=> new("DATA_FORMAT_ERROR", VexWorkerErrorCategory.DataFormat, message, retryable: false);
|
||||
|
||||
/// <summary>
|
||||
/// Creates an internal error.
|
||||
/// </summary>
|
||||
public static VexWorkerError Internal(string message)
|
||||
=> new("INTERNAL_ERROR", VexWorkerErrorCategory.Internal, message, retryable: false);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a timeout error.
|
||||
/// </summary>
|
||||
public static VexWorkerError Timeout(string message, int? retryAfterSeconds = 30)
|
||||
=> new("TIMEOUT", VexWorkerErrorCategory.Timeout, message, retryable: true, retryAfterSeconds);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a cancelled error.
|
||||
/// </summary>
|
||||
public static VexWorkerError Cancelled(string message)
|
||||
=> new("CANCELLED", VexWorkerErrorCategory.Cancelled, message, retryable: false);
|
||||
|
||||
/// <summary>
|
||||
/// Classifies an exception into an appropriate error.
|
||||
/// </summary>
|
||||
public static VexWorkerError FromException(Exception ex, string? stage = null)
|
||||
{
|
||||
return ex switch
|
||||
{
|
||||
OperationCanceledException => Cancelled(ex.Message),
|
||||
TimeoutException => Timeout(ex.Message),
|
||||
System.Net.Http.HttpRequestException httpEx when httpEx.StatusCode == System.Net.HttpStatusCode.TooManyRequests
|
||||
=> RateLimited(ex.Message, 60),
|
||||
System.Net.Http.HttpRequestException httpEx when httpEx.StatusCode == System.Net.HttpStatusCode.Unauthorized
|
||||
|| httpEx.StatusCode == System.Net.HttpStatusCode.Forbidden
|
||||
=> Authorization(ex.Message),
|
||||
System.Net.Http.HttpRequestException httpEx when httpEx.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable
|
||||
|| httpEx.StatusCode == System.Net.HttpStatusCode.BadGateway
|
||||
|| httpEx.StatusCode == System.Net.HttpStatusCode.GatewayTimeout
|
||||
=> ServiceUnavailable(ex.Message),
|
||||
System.Net.Http.HttpRequestException => Network(ex.Message),
|
||||
System.Net.Sockets.SocketException => Network(ex.Message),
|
||||
System.IO.IOException => Network(ex.Message),
|
||||
System.Text.Json.JsonException => DataFormat(ex.Message),
|
||||
FormatException => DataFormat(ex.Message),
|
||||
InvalidOperationException => Internal(ex.Message),
|
||||
_ => new VexWorkerError("UNKNOWN_ERROR", VexWorkerErrorCategory.Unknown, ex.Message, retryable: false, stage: stage)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checkpoint state for resumable ingestion.
|
||||
/// </summary>
|
||||
public sealed record VexWorkerCheckpoint(
|
||||
string ConnectorId,
|
||||
string? Cursor,
|
||||
DateTimeOffset? LastProcessedAt,
|
||||
ImmutableArray<string> ProcessedDigests,
|
||||
ImmutableDictionary<string, string> ResumeTokens)
|
||||
{
|
||||
public static VexWorkerCheckpoint Empty(string connectorId) => new(
|
||||
connectorId,
|
||||
Cursor: null,
|
||||
LastProcessedAt: null,
|
||||
ProcessedDigests: ImmutableArray<string>.Empty,
|
||||
ResumeTokens: ImmutableDictionary<string, string>.Empty);
|
||||
}
|
||||
@@ -124,7 +124,16 @@ public sealed class OpenVexExporter : IVexExporter
|
||||
SourceUri: source.DocumentSource.ToString(),
|
||||
Detail: source.Detail,
|
||||
FirstObserved: source.FirstSeen.UtcDateTime.ToString("O", CultureInfo.InvariantCulture),
|
||||
LastObserved: source.LastSeen.UtcDateTime.ToString("O", CultureInfo.InvariantCulture)))
|
||||
LastObserved: source.LastSeen.UtcDateTime.ToString("O", CultureInfo.InvariantCulture),
|
||||
// VEX Lens enrichment fields
|
||||
IssuerHint: source.IssuerHint,
|
||||
SignatureType: source.SignatureType,
|
||||
KeyId: source.KeyId,
|
||||
TransparencyLogRef: source.TransparencyLogRef,
|
||||
TrustWeight: source.TrustWeight,
|
||||
TrustTier: source.TrustTier,
|
||||
StalenessSeconds: source.StalenessSeconds,
|
||||
ProductTreeSnippet: source.ProductTreeSnippet))
|
||||
.ToImmutableArray();
|
||||
|
||||
var statementId = FormattableString.Invariant($"{statement.VulnerabilityId}#{NormalizeProductKey(statement.Product.Key)}");
|
||||
@@ -200,6 +209,9 @@ internal sealed record OpenVexExportProduct(
|
||||
[property: JsonPropertyName("purl"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Purl,
|
||||
[property: JsonPropertyName("cpe"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Cpe);
|
||||
|
||||
/// <summary>
|
||||
/// OpenVEX source entry with VEX Lens enrichment fields for consensus computation.
|
||||
/// </summary>
|
||||
internal sealed record OpenVexExportSource(
|
||||
[property: JsonPropertyName("provider")] string Provider,
|
||||
[property: JsonPropertyName("status")] string Status,
|
||||
@@ -208,7 +220,16 @@ internal sealed record OpenVexExportSource(
|
||||
[property: JsonPropertyName("source_uri")] string SourceUri,
|
||||
[property: JsonPropertyName("detail"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Detail,
|
||||
[property: JsonPropertyName("first_observed")] string FirstObserved,
|
||||
[property: JsonPropertyName("last_observed")] string LastObserved);
|
||||
[property: JsonPropertyName("last_observed")] string LastObserved,
|
||||
// VEX Lens enrichment fields for consensus without callback to Excititor
|
||||
[property: JsonPropertyName("issuer_hint"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? IssuerHint,
|
||||
[property: JsonPropertyName("signature_type"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? SignatureType,
|
||||
[property: JsonPropertyName("key_id"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? KeyId,
|
||||
[property: JsonPropertyName("transparency_log_ref"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? TransparencyLogRef,
|
||||
[property: JsonPropertyName("trust_weight"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] decimal? TrustWeight,
|
||||
[property: JsonPropertyName("trust_tier"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? TrustTier,
|
||||
[property: JsonPropertyName("staleness_seconds"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] long? StalenessSeconds,
|
||||
[property: JsonPropertyName("product_tree_snippet"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ProductTreeSnippet);
|
||||
|
||||
internal sealed record OpenVexExportMetadata(
|
||||
[property: JsonPropertyName("generated_at")] string GeneratedAt,
|
||||
|
||||
@@ -169,17 +169,60 @@ public static class OpenVexStatementMerger
|
||||
private static ImmutableArray<OpenVexSourceEntry> BuildSources(ImmutableArray<VexClaim> claims)
|
||||
{
|
||||
var builder = ImmutableArray.CreateBuilder<OpenVexSourceEntry>(claims.Length);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
|
||||
foreach (var claim in claims)
|
||||
{
|
||||
// Extract VEX Lens enrichment from signature metadata
|
||||
var signature = claim.Document.Signature;
|
||||
var trust = signature?.Trust;
|
||||
|
||||
// Compute staleness from trust metadata retrieval time or last seen
|
||||
long? stalenessSeconds = null;
|
||||
if (trust?.RetrievedAtUtc is { } retrievedAt)
|
||||
{
|
||||
stalenessSeconds = (long)Math.Ceiling((now - retrievedAt).TotalSeconds);
|
||||
}
|
||||
else if (signature?.VerifiedAt is { } verifiedAt)
|
||||
{
|
||||
stalenessSeconds = (long)Math.Ceiling((now - verifiedAt).TotalSeconds);
|
||||
}
|
||||
|
||||
// Extract product tree snippet from additional metadata (if present)
|
||||
string? productTreeSnippet = null;
|
||||
if (claim.AdditionalMetadata.TryGetValue("csaf.product_tree", out var productTree))
|
||||
{
|
||||
productTreeSnippet = productTree;
|
||||
}
|
||||
|
||||
// Derive trust tier from issuer or provider type
|
||||
string? trustTier = null;
|
||||
if (trust is not null)
|
||||
{
|
||||
trustTier = trust.TenantOverrideApplied ? "tenant-override" : DeriveIssuerTier(trust.IssuerId);
|
||||
}
|
||||
else if (claim.AdditionalMetadata.TryGetValue("issuer.tier", out var tier))
|
||||
{
|
||||
trustTier = tier;
|
||||
}
|
||||
|
||||
builder.Add(new OpenVexSourceEntry(
|
||||
claim.ProviderId,
|
||||
claim.Status,
|
||||
claim.Justification,
|
||||
claim.Document.Digest,
|
||||
claim.Document.SourceUri,
|
||||
claim.Detail,
|
||||
claim.FirstSeen,
|
||||
claim.LastSeen));
|
||||
providerId: claim.ProviderId,
|
||||
status: claim.Status,
|
||||
justification: claim.Justification,
|
||||
documentDigest: claim.Document.Digest,
|
||||
documentSource: claim.Document.SourceUri,
|
||||
detail: claim.Detail,
|
||||
firstSeen: claim.FirstSeen,
|
||||
lastSeen: claim.LastSeen,
|
||||
issuerHint: signature?.Issuer ?? signature?.Subject,
|
||||
signatureType: signature?.Type,
|
||||
keyId: signature?.KeyId,
|
||||
transparencyLogRef: signature?.TransparencyLogReference,
|
||||
trustWeight: trust?.EffectiveWeight,
|
||||
trustTier: trustTier,
|
||||
stalenessSeconds: stalenessSeconds,
|
||||
productTreeSnippet: productTreeSnippet));
|
||||
}
|
||||
|
||||
return builder
|
||||
@@ -189,6 +232,34 @@ public static class OpenVexStatementMerger
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
private static string? DeriveIssuerTier(string issuerId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(issuerId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// Common issuer tier patterns
|
||||
var lowerIssuerId = issuerId.ToLowerInvariant();
|
||||
if (lowerIssuerId.Contains("vendor") || lowerIssuerId.Contains("upstream"))
|
||||
{
|
||||
return "vendor";
|
||||
}
|
||||
|
||||
if (lowerIssuerId.Contains("distro") || lowerIssuerId.Contains("rhel") ||
|
||||
lowerIssuerId.Contains("ubuntu") || lowerIssuerId.Contains("debian"))
|
||||
{
|
||||
return "distro-trusted";
|
||||
}
|
||||
|
||||
if (lowerIssuerId.Contains("community") || lowerIssuerId.Contains("oss"))
|
||||
{
|
||||
return "community";
|
||||
}
|
||||
|
||||
return "other";
|
||||
}
|
||||
|
||||
private static VexProduct MergeProduct(ImmutableArray<VexClaim> claims)
|
||||
{
|
||||
var key = claims[0].Product.Key;
|
||||
@@ -266,17 +337,85 @@ public sealed record OpenVexMergedStatement(
|
||||
DateTimeOffset FirstObserved,
|
||||
DateTimeOffset LastObserved);
|
||||
|
||||
public sealed record OpenVexSourceEntry(
|
||||
string ProviderId,
|
||||
VexClaimStatus Status,
|
||||
VexJustification? Justification,
|
||||
string DocumentDigest,
|
||||
Uri DocumentSource,
|
||||
string? Detail,
|
||||
DateTimeOffset FirstSeen,
|
||||
DateTimeOffset LastSeen)
|
||||
/// <summary>
|
||||
/// Represents a merged VEX source entry with enrichment for VEX Lens consumption.
|
||||
/// </summary>
|
||||
public sealed record OpenVexSourceEntry
|
||||
{
|
||||
public string DocumentDigest { get; } = string.IsNullOrWhiteSpace(DocumentDigest)
|
||||
? throw new ArgumentException("Document digest must be provided.", nameof(DocumentDigest))
|
||||
: DocumentDigest.Trim();
|
||||
public OpenVexSourceEntry(
|
||||
string providerId,
|
||||
VexClaimStatus status,
|
||||
VexJustification? justification,
|
||||
string documentDigest,
|
||||
Uri documentSource,
|
||||
string? detail,
|
||||
DateTimeOffset firstSeen,
|
||||
DateTimeOffset lastSeen,
|
||||
string? issuerHint = null,
|
||||
string? signatureType = null,
|
||||
string? keyId = null,
|
||||
string? transparencyLogRef = null,
|
||||
decimal? trustWeight = null,
|
||||
string? trustTier = null,
|
||||
long? stalenessSeconds = null,
|
||||
string? productTreeSnippet = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(documentDigest))
|
||||
{
|
||||
throw new ArgumentException("Document digest must be provided.", nameof(documentDigest));
|
||||
}
|
||||
|
||||
ProviderId = providerId;
|
||||
Status = status;
|
||||
Justification = justification;
|
||||
DocumentDigest = documentDigest.Trim();
|
||||
DocumentSource = documentSource;
|
||||
Detail = detail;
|
||||
FirstSeen = firstSeen;
|
||||
LastSeen = lastSeen;
|
||||
|
||||
// VEX Lens enrichment fields
|
||||
IssuerHint = string.IsNullOrWhiteSpace(issuerHint) ? null : issuerHint.Trim();
|
||||
SignatureType = string.IsNullOrWhiteSpace(signatureType) ? null : signatureType.Trim();
|
||||
KeyId = string.IsNullOrWhiteSpace(keyId) ? null : keyId.Trim();
|
||||
TransparencyLogRef = string.IsNullOrWhiteSpace(transparencyLogRef) ? null : transparencyLogRef.Trim();
|
||||
TrustWeight = trustWeight;
|
||||
TrustTier = string.IsNullOrWhiteSpace(trustTier) ? null : trustTier.Trim();
|
||||
StalenessSeconds = stalenessSeconds;
|
||||
ProductTreeSnippet = string.IsNullOrWhiteSpace(productTreeSnippet) ? null : productTreeSnippet.Trim();
|
||||
}
|
||||
|
||||
public string ProviderId { get; }
|
||||
public VexClaimStatus Status { get; }
|
||||
public VexJustification? Justification { get; }
|
||||
public string DocumentDigest { get; }
|
||||
public Uri DocumentSource { get; }
|
||||
public string? Detail { get; }
|
||||
public DateTimeOffset FirstSeen { get; }
|
||||
public DateTimeOffset LastSeen { get; }
|
||||
|
||||
// VEX Lens enrichment fields for consensus computation
|
||||
/// <summary>Issuer identity/hint (e.g., vendor name, distro-trusted) for trust weighting.</summary>
|
||||
public string? IssuerHint { get; }
|
||||
|
||||
/// <summary>Cryptographic signature type (jws, pgp, cosign, etc.).</summary>
|
||||
public string? SignatureType { get; }
|
||||
|
||||
/// <summary>Key identifier used for signature verification.</summary>
|
||||
public string? KeyId { get; }
|
||||
|
||||
/// <summary>Transparency log reference (e.g., Rekor URL) for attestation verification.</summary>
|
||||
public string? TransparencyLogRef { get; }
|
||||
|
||||
/// <summary>Trust weight (0-1) from issuer directory for consensus calculation.</summary>
|
||||
public decimal? TrustWeight { get; }
|
||||
|
||||
/// <summary>Trust tier label (vendor, distro-trusted, community, etc.).</summary>
|
||||
public string? TrustTier { get; }
|
||||
|
||||
/// <summary>Seconds since the document was last verified/retrieved.</summary>
|
||||
public long? StalenessSeconds { get; }
|
||||
|
||||
/// <summary>Product tree snippet (JSON) from CSAF documents for product matching.</summary>
|
||||
public string? ProductTreeSnippet { get; }
|
||||
}
|
||||
|
||||
@@ -17,17 +17,17 @@ public interface IVexProviderStore
|
||||
ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexConsensusStore
|
||||
{
|
||||
ValueTask<VexConsensus?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexConsensus>> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
IAsyncEnumerable<VexConsensus> FindCalculatedBeforeAsync(DateTimeOffset cutoff, int batchSize, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
public interface IVexConsensusStore
|
||||
{
|
||||
ValueTask<VexConsensus?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexConsensus>> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
IAsyncEnumerable<VexConsensus> FindCalculatedBeforeAsync(DateTimeOffset cutoff, int batchSize, CancellationToken cancellationToken, IClientSessionHandle? session = null)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public interface IVexClaimStore
|
||||
{
|
||||
@@ -44,7 +44,12 @@ public sealed record VexConnectorState(
|
||||
DateTimeOffset? LastSuccessAt,
|
||||
int FailureCount,
|
||||
DateTimeOffset? NextEligibleRun,
|
||||
string? LastFailureReason)
|
||||
string? LastFailureReason,
|
||||
DateTimeOffset? LastHeartbeatAt = null,
|
||||
string? LastHeartbeatStatus = null,
|
||||
string? LastArtifactHash = null,
|
||||
string? LastArtifactKind = null,
|
||||
string? LastCheckpoint = null)
|
||||
{
|
||||
public VexConnectorState(
|
||||
string connectorId,
|
||||
@@ -58,30 +63,35 @@ public sealed record VexConnectorState(
|
||||
LastSuccessAt: null,
|
||||
FailureCount: 0,
|
||||
NextEligibleRun: null,
|
||||
LastFailureReason: null)
|
||||
LastFailureReason: null,
|
||||
LastHeartbeatAt: null,
|
||||
LastHeartbeatStatus: null,
|
||||
LastArtifactHash: null,
|
||||
LastArtifactKind: null,
|
||||
LastCheckpoint: null)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public interface IVexConnectorStateRepository
|
||||
{
|
||||
ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexConsensusHoldStore
|
||||
{
|
||||
ValueTask<VexConsensusHold?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConsensusHold hold, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask RemoveAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
IAsyncEnumerable<VexConsensusHold> FindEligibleAsync(DateTimeOffset asOf, int batchSize, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
public interface IVexConnectorStateRepository
|
||||
{
|
||||
ValueTask<VexConnectorState?> GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexConsensusHoldStore
|
||||
{
|
||||
ValueTask<VexConsensusHold?> FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask SaveAsync(VexConsensusHold hold, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
ValueTask RemoveAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
|
||||
IAsyncEnumerable<VexConsensusHold> FindEligibleAsync(DateTimeOffset asOf, int batchSize, CancellationToken cancellationToken, IClientSessionHandle? session = null);
|
||||
}
|
||||
|
||||
public interface IVexCacheIndex
|
||||
{
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
/// <summary>
|
||||
/// Adds idempotency indexes to the vex_raw collection to enforce content-addressed storage.
|
||||
/// Ensures that:
|
||||
/// 1. Each document is uniquely identified by its content digest
|
||||
/// 2. Provider+Source combinations are unique per digest
|
||||
/// 3. Supports efficient queries for evidence retrieval
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Rollback: Run db.vex_raw.dropIndex("idx_provider_sourceUri_digest_unique")
|
||||
/// and db.vex_raw.dropIndex("idx_digest_providerId") to reverse this migration.
|
||||
/// </remarks>
|
||||
internal sealed class VexRawIdempotencyIndexMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251127-vex-raw-idempotency-indexes";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
|
||||
// Index 1: Unique constraint on providerId + sourceUri + digest
|
||||
// Ensures the same document from the same provider/source is only stored once
|
||||
var providerSourceDigestIndex = new BsonDocument
|
||||
{
|
||||
{ "providerId", 1 },
|
||||
{ "sourceUri", 1 },
|
||||
{ "digest", 1 }
|
||||
};
|
||||
|
||||
var uniqueIndexModel = new CreateIndexModel<BsonDocument>(
|
||||
providerSourceDigestIndex,
|
||||
new CreateIndexOptions
|
||||
{
|
||||
Unique = true,
|
||||
Name = "idx_provider_sourceUri_digest_unique",
|
||||
Background = true
|
||||
});
|
||||
|
||||
// Index 2: Compound index for efficient evidence queries by digest + provider
|
||||
var digestProviderIndex = new BsonDocument
|
||||
{
|
||||
{ "digest", 1 },
|
||||
{ "providerId", 1 }
|
||||
};
|
||||
|
||||
var queryIndexModel = new CreateIndexModel<BsonDocument>(
|
||||
digestProviderIndex,
|
||||
new CreateIndexOptions
|
||||
{
|
||||
Name = "idx_digest_providerId",
|
||||
Background = true
|
||||
});
|
||||
|
||||
// Index 3: TTL index candidate for future cleanup (optional staleness tracking)
|
||||
var retrievedAtIndex = new BsonDocument
|
||||
{
|
||||
{ "retrievedAt", 1 }
|
||||
};
|
||||
|
||||
var retrievedAtIndexModel = new CreateIndexModel<BsonDocument>(
|
||||
retrievedAtIndex,
|
||||
new CreateIndexOptions
|
||||
{
|
||||
Name = "idx_retrievedAt",
|
||||
Background = true
|
||||
});
|
||||
|
||||
// Create all indexes
|
||||
await collection.Indexes.CreateManyAsync(
|
||||
new[] { uniqueIndexModel, queryIndexModel, retrievedAtIndexModel },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for idempotency index management.
|
||||
/// </summary>
|
||||
public static class VexRawIdempotencyIndexExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Drops the idempotency indexes (for rollback).
|
||||
/// </summary>
|
||||
public static async Task RollbackIdempotencyIndexesAsync(
|
||||
this IMongoDatabase database,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
|
||||
var indexNames = new[]
|
||||
{
|
||||
"idx_provider_sourceUri_digest_unique",
|
||||
"idx_digest_providerId",
|
||||
"idx_retrievedAt"
|
||||
};
|
||||
|
||||
foreach (var indexName in indexNames)
|
||||
{
|
||||
try
|
||||
{
|
||||
await collection.Indexes.DropOneAsync(indexName, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (MongoCommandException ex) when (ex.CodeName == "IndexNotFound")
|
||||
{
|
||||
// Index doesn't exist, skip
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that idempotency indexes exist.
|
||||
/// </summary>
|
||||
public static async Task<bool> VerifyIdempotencyIndexesExistAsync(
|
||||
this IMongoDatabase database,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
|
||||
var cursor = await collection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false);
|
||||
var indexes = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var indexNames = indexes.Select(i => i.GetValue("name", "").AsString).ToHashSet();
|
||||
|
||||
return indexNames.Contains("idx_provider_sourceUri_digest_unique") &&
|
||||
indexNames.Contains("idx_digest_providerId");
|
||||
}
|
||||
}
|
||||
@@ -25,15 +25,17 @@ internal sealed class VexRawSchemaMigration : IVexMongoMigration
|
||||
|
||||
if (!exists)
|
||||
{
|
||||
await database.CreateCollectionAsync(
|
||||
VexMongoCollectionNames.Raw,
|
||||
new CreateCollectionOptions
|
||||
{
|
||||
Validator = validator,
|
||||
ValidationAction = DocumentValidationAction.Warn,
|
||||
ValidationLevel = DocumentValidationLevel.Moderate,
|
||||
},
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
// In MongoDB.Driver 3.x, CreateCollectionOptions doesn't support Validator directly.
|
||||
// Use the create command instead.
|
||||
var createCommand = new BsonDocument
|
||||
{
|
||||
{ "create", VexMongoCollectionNames.Raw },
|
||||
{ "validator", validator },
|
||||
{ "validationAction", "warn" },
|
||||
{ "validationLevel", "moderate" }
|
||||
};
|
||||
await database.RunCommandAsync<BsonDocument>(createCommand, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using MongoDB.Driver;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
|
||||
/// <summary>
|
||||
/// Migration that creates indexes for the vex.timeline_events collection.
|
||||
/// </summary>
|
||||
internal sealed class VexTimelineEventIndexMigration : IVexMongoMigration
|
||||
{
|
||||
public string Id => "20251127-timeline-events";
|
||||
|
||||
public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<VexTimelineEventRecord>(VexMongoCollectionNames.TimelineEvents);
|
||||
|
||||
// Unique index on tenant + event ID
|
||||
var tenantEventIdIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.Id);
|
||||
|
||||
// Index for querying by time range (descending for recent-first queries)
|
||||
var tenantTimeIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Descending(x => x.CreatedAt);
|
||||
|
||||
// Index for querying by trace ID
|
||||
var tenantTraceIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.TraceId)
|
||||
.Ascending(x => x.CreatedAt);
|
||||
|
||||
// Index for querying by provider
|
||||
var tenantProviderIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.ProviderId)
|
||||
.Descending(x => x.CreatedAt);
|
||||
|
||||
// Index for querying by event type
|
||||
var tenantEventTypeIndex = Builders<VexTimelineEventRecord>.IndexKeys
|
||||
.Ascending(x => x.Tenant)
|
||||
.Ascending(x => x.EventType)
|
||||
.Descending(x => x.CreatedAt);
|
||||
|
||||
// TTL index for automatic cleanup (30 days by default)
|
||||
// Uncomment if timeline events should expire:
|
||||
// var ttlIndex = Builders<VexTimelineEventRecord>.IndexKeys.Ascending(x => x.CreatedAt);
|
||||
// var ttlOptions = new CreateIndexOptions { ExpireAfter = TimeSpan.FromDays(30) };
|
||||
|
||||
await Task.WhenAll(
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantEventIdIndex, new CreateIndexOptions { Unique = true }),
|
||||
cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantTimeIndex),
|
||||
cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantTraceIndex),
|
||||
cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantProviderIndex),
|
||||
cancellationToken: cancellationToken),
|
||||
collection.Indexes.CreateOneAsync(
|
||||
new CreateIndexModel<VexTimelineEventRecord>(tenantEventTypeIndex),
|
||||
cancellationToken: cancellationToken)
|
||||
).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB implementation of <see cref="IVexLinksetEventPublisher"/>.
|
||||
/// Events are persisted to the vex.linkset_events collection for replay and audit.
|
||||
/// </summary>
|
||||
internal sealed class MongoVexLinksetEventPublisher : IVexLinksetEventPublisher
|
||||
{
|
||||
private readonly IMongoCollection<VexLinksetEventRecord> _collection;
|
||||
|
||||
public MongoVexLinksetEventPublisher(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexLinksetEventRecord>(VexMongoCollectionNames.LinksetEvents);
|
||||
}
|
||||
|
||||
public async Task PublishAsync(VexLinksetUpdatedEvent @event, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(@event);
|
||||
|
||||
var record = ToRecord(@event);
|
||||
await _collection.InsertOneAsync(record, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task PublishManyAsync(IEnumerable<VexLinksetUpdatedEvent> events, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(events);
|
||||
|
||||
var records = events
|
||||
.Where(e => e is not null)
|
||||
.Select(ToRecord)
|
||||
.ToList();
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var options = new InsertManyOptions { IsOrdered = false };
|
||||
await _collection.InsertManyAsync(records, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static VexLinksetEventRecord ToRecord(VexLinksetUpdatedEvent @event)
|
||||
{
|
||||
var eventId = $"{@event.LinksetId}:{@event.CreatedAtUtc.UtcTicks}";
|
||||
|
||||
return new VexLinksetEventRecord
|
||||
{
|
||||
Id = eventId,
|
||||
EventType = @event.EventType,
|
||||
Tenant = @event.Tenant.ToLowerInvariant(),
|
||||
LinksetId = @event.LinksetId,
|
||||
VulnerabilityId = @event.VulnerabilityId,
|
||||
ProductKey = @event.ProductKey,
|
||||
Observations = @event.Observations
|
||||
.Select(o => new VexLinksetEventObservationRecord
|
||||
{
|
||||
ObservationId = o.ObservationId,
|
||||
ProviderId = o.ProviderId,
|
||||
Status = o.Status,
|
||||
Confidence = o.Confidence
|
||||
})
|
||||
.ToList(),
|
||||
Disagreements = @event.Disagreements
|
||||
.Select(d => new VexLinksetDisagreementRecord
|
||||
{
|
||||
ProviderId = d.ProviderId,
|
||||
Status = d.Status,
|
||||
Justification = d.Justification,
|
||||
Confidence = d.Confidence
|
||||
})
|
||||
.ToList(),
|
||||
CreatedAtUtc = @event.CreatedAtUtc.UtcDateTime,
|
||||
PublishedAtUtc = DateTime.UtcNow,
|
||||
ConflictCount = @event.Disagreements.Length,
|
||||
ObservationCount = @event.Observations.Length
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
using System.Collections.Immutable;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
internal sealed class MongoVexLinksetStore : IVexLinksetStore
|
||||
{
|
||||
private readonly IMongoCollection<VexLinksetRecord> _collection;
|
||||
|
||||
public MongoVexLinksetStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexLinksetRecord>(VexMongoCollectionNames.Linksets);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> InsertAsync(
|
||||
VexLinkset linkset,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(linkset);
|
||||
|
||||
var record = ToRecord(linkset);
|
||||
|
||||
try
|
||||
{
|
||||
await _collection.InsertOneAsync(record, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<bool> UpsertAsync(
|
||||
VexLinkset linkset,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(linkset);
|
||||
|
||||
var record = ToRecord(linkset);
|
||||
var normalizedTenant = NormalizeTenant(linkset.Tenant);
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.LinksetId, linkset.LinksetId));
|
||||
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
var result = await _collection
|
||||
.ReplaceOneAsync(filter, record, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return result.UpsertedId is not null;
|
||||
}
|
||||
|
||||
public async ValueTask<VexLinkset?> GetByIdAsync(
|
||||
string tenant,
|
||||
string linksetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = linksetId?.Trim() ?? throw new ArgumentNullException(nameof(linksetId));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.LinksetId, normalizedId));
|
||||
|
||||
var record = await _collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return record is null ? null : ToModel(record);
|
||||
}
|
||||
|
||||
public async ValueTask<VexLinkset> GetOrCreateAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedVuln = vulnerabilityId?.Trim() ?? throw new ArgumentNullException(nameof(vulnerabilityId));
|
||||
var normalizedProduct = productKey?.Trim() ?? throw new ArgumentNullException(nameof(productKey));
|
||||
|
||||
var linksetId = VexLinkset.CreateLinksetId(normalizedTenant, normalizedVuln, normalizedProduct);
|
||||
|
||||
var existing = await GetByIdAsync(normalizedTenant, linksetId, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null)
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
var newLinkset = new VexLinkset(
|
||||
linksetId,
|
||||
normalizedTenant,
|
||||
normalizedVuln,
|
||||
normalizedProduct,
|
||||
observations: Array.Empty<VexLinksetObservationRefModel>(),
|
||||
disagreements: null,
|
||||
createdAt: DateTimeOffset.UtcNow,
|
||||
updatedAt: DateTimeOffset.UtcNow);
|
||||
|
||||
try
|
||||
{
|
||||
await InsertAsync(newLinkset, cancellationToken).ConfigureAwait(false);
|
||||
return newLinkset;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
// Race condition - another process created it. Fetch and return.
|
||||
var created = await GetByIdAsync(normalizedTenant, linksetId, cancellationToken).ConfigureAwait(false);
|
||||
return created ?? newLinkset;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexLinkset>> FindByVulnerabilityAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedVuln = vulnerabilityId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(vulnerabilityId));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.VulnerabilityId, normalizedVuln));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexLinksetRecord>.Sort.Descending(r => r.UpdatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexLinkset>> FindByProductKeyAsync(
|
||||
string tenant,
|
||||
string productKey,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedProduct = productKey?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(productKey));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.ProductKey, normalizedProduct));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexLinksetRecord>.Sort.Descending(r => r.UpdatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexLinkset>> FindWithConflictsAsync(
|
||||
string tenant,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.SizeGt(r => r.Disagreements, 0));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexLinksetRecord>.Sort.Descending(r => r.UpdatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexLinkset>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedProvider = providerId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(providerId));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.AnyEq(r => r.ProviderIds, normalizedProvider));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexLinksetRecord>.Sort.Descending(r => r.UpdatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<bool> DeleteAsync(
|
||||
string tenant,
|
||||
string linksetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = linksetId?.Trim() ?? throw new ArgumentNullException(nameof(linksetId));
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.LinksetId, normalizedId));
|
||||
|
||||
var result = await _collection
|
||||
.DeleteOneAsync(filter, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return result.DeletedCount > 0;
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant);
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountWithConflictsAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexLinksetRecord>.Filter.And(
|
||||
Builders<VexLinksetRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexLinksetRecord>.Filter.SizeGt(r => r.Disagreements, 0));
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string NormalizeTenant(string tenant)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static VexLinksetRecord ToRecord(VexLinkset linkset)
|
||||
{
|
||||
return new VexLinksetRecord
|
||||
{
|
||||
Id = linkset.LinksetId,
|
||||
Tenant = linkset.Tenant.ToLowerInvariant(),
|
||||
LinksetId = linkset.LinksetId,
|
||||
VulnerabilityId = linkset.VulnerabilityId.ToLowerInvariant(),
|
||||
ProductKey = linkset.ProductKey.ToLowerInvariant(),
|
||||
ProviderIds = linkset.ProviderIds.ToList(),
|
||||
Statuses = linkset.Statuses.ToList(),
|
||||
CreatedAt = linkset.CreatedAt.UtcDateTime,
|
||||
UpdatedAt = linkset.UpdatedAt.UtcDateTime,
|
||||
Observations = linkset.Observations.Select(ToObservationRecord).ToList(),
|
||||
Disagreements = linkset.Disagreements.Select(ToDisagreementRecord).ToList()
|
||||
};
|
||||
}
|
||||
|
||||
private static VexObservationLinksetObservationRecord ToObservationRecord(VexLinksetObservationRefModel obs)
|
||||
{
|
||||
return new VexObservationLinksetObservationRecord
|
||||
{
|
||||
ObservationId = obs.ObservationId,
|
||||
ProviderId = obs.ProviderId,
|
||||
Status = obs.Status,
|
||||
Confidence = obs.Confidence
|
||||
};
|
||||
}
|
||||
|
||||
private static VexLinksetDisagreementRecord ToDisagreementRecord(VexObservationDisagreement disagreement)
|
||||
{
|
||||
return new VexLinksetDisagreementRecord
|
||||
{
|
||||
ProviderId = disagreement.ProviderId,
|
||||
Status = disagreement.Status,
|
||||
Justification = disagreement.Justification,
|
||||
Confidence = disagreement.Confidence
|
||||
};
|
||||
}
|
||||
|
||||
private static VexLinkset ToModel(VexLinksetRecord record)
|
||||
{
|
||||
var observations = record.Observations?
|
||||
.Where(o => o is not null)
|
||||
.Select(o => new VexLinksetObservationRefModel(
|
||||
o.ObservationId,
|
||||
o.ProviderId,
|
||||
o.Status,
|
||||
o.Confidence))
|
||||
.ToImmutableArray() ?? ImmutableArray<VexLinksetObservationRefModel>.Empty;
|
||||
|
||||
var disagreements = record.Disagreements?
|
||||
.Where(d => d is not null)
|
||||
.Select(d => new VexObservationDisagreement(
|
||||
d.ProviderId,
|
||||
d.Status,
|
||||
d.Justification,
|
||||
d.Confidence))
|
||||
.ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
|
||||
return new VexLinkset(
|
||||
linksetId: record.LinksetId,
|
||||
tenant: record.Tenant,
|
||||
vulnerabilityId: record.VulnerabilityId,
|
||||
productKey: record.ProductKey,
|
||||
observations: observations,
|
||||
disagreements: disagreements,
|
||||
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
|
||||
updatedAt: new DateTimeOffset(record.UpdatedAt, TimeSpan.Zero));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Nodes;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
internal sealed class MongoVexObservationStore : IVexObservationStore
|
||||
{
|
||||
private readonly IMongoCollection<VexObservationRecord> _collection;
|
||||
|
||||
public MongoVexObservationStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexObservationRecord>(VexMongoCollectionNames.Observations);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> InsertAsync(
|
||||
VexObservation observation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(observation);
|
||||
|
||||
var record = ToRecord(observation);
|
||||
|
||||
try
|
||||
{
|
||||
await _collection.InsertOneAsync(record, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<bool> UpsertAsync(
|
||||
VexObservation observation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(observation);
|
||||
|
||||
var record = ToRecord(observation);
|
||||
var normalizedTenant = NormalizeTenant(observation.Tenant);
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ObservationId, observation.ObservationId));
|
||||
|
||||
var options = new ReplaceOptions { IsUpsert = true };
|
||||
var result = await _collection
|
||||
.ReplaceOneAsync(filter, record, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return result.UpsertedId is not null;
|
||||
}
|
||||
|
||||
public async ValueTask<int> InsertManyAsync(
|
||||
string tenant,
|
||||
IEnumerable<VexObservation> observations,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var records = observations
|
||||
.Where(o => o is not null && string.Equals(NormalizeTenant(o.Tenant), normalizedTenant, StringComparison.Ordinal))
|
||||
.Select(ToRecord)
|
||||
.ToList();
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var options = new InsertManyOptions { IsOrdered = false };
|
||||
try
|
||||
{
|
||||
await _collection.InsertManyAsync(records, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return records.Count;
|
||||
}
|
||||
catch (MongoBulkWriteException<VexObservationRecord> ex)
|
||||
{
|
||||
// Return the count of successful inserts
|
||||
var duplicates = ex.WriteErrors?.Count(e => e.Category == ServerErrorCategory.DuplicateKey) ?? 0;
|
||||
return records.Count - duplicates;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<VexObservation?> GetByIdAsync(
|
||||
string tenant,
|
||||
string observationId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = observationId?.Trim() ?? throw new ArgumentNullException(nameof(observationId));
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ObservationId, normalizedId));
|
||||
|
||||
var record = await _collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return record is null ? null : ToModel(record);
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexObservation>> FindByVulnerabilityAndProductAsync(
|
||||
string tenant,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedVuln = vulnerabilityId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(vulnerabilityId));
|
||||
var normalizedProduct = productKey?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(productKey));
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.VulnerabilityId, normalizedVuln),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ProductKey, normalizedProduct));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<VexObservation>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedProvider = providerId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(providerId));
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ProviderId, normalizedProvider));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexObservationRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<bool> DeleteAsync(
|
||||
string tenant,
|
||||
string observationId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = observationId?.Trim() ?? throw new ArgumentNullException(nameof(observationId));
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.And(
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexObservationRecord>.Filter.Eq(r => r.ObservationId, normalizedId));
|
||||
|
||||
var result = await _collection
|
||||
.DeleteOneAsync(filter, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return result.DeletedCount > 0;
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexObservationRecord>.Filter.Eq(r => r.Tenant, normalizedTenant);
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string NormalizeTenant(string tenant)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static VexObservationRecord ToRecord(VexObservation observation)
|
||||
{
|
||||
var firstStatement = observation.Statements.FirstOrDefault();
|
||||
|
||||
return new VexObservationRecord
|
||||
{
|
||||
Id = observation.ObservationId,
|
||||
Tenant = observation.Tenant,
|
||||
ObservationId = observation.ObservationId,
|
||||
VulnerabilityId = firstStatement?.VulnerabilityId?.ToLowerInvariant() ?? string.Empty,
|
||||
ProductKey = firstStatement?.ProductKey?.ToLowerInvariant() ?? string.Empty,
|
||||
ProviderId = observation.ProviderId,
|
||||
StreamId = observation.StreamId,
|
||||
Status = firstStatement?.Status.ToString().ToLowerInvariant() ?? "unknown",
|
||||
Document = new VexObservationDocumentRecord
|
||||
{
|
||||
Digest = observation.Upstream.ContentHash,
|
||||
SourceUri = null,
|
||||
Format = observation.Content.Format,
|
||||
Revision = observation.Upstream.DocumentVersion,
|
||||
Signature = new VexObservationSignatureRecord
|
||||
{
|
||||
Present = observation.Upstream.Signature.Present,
|
||||
Subject = observation.Upstream.Signature.Format,
|
||||
Issuer = observation.Upstream.Signature.KeyId,
|
||||
VerifiedAt = null
|
||||
}
|
||||
},
|
||||
Upstream = new VexObservationUpstreamRecord
|
||||
{
|
||||
UpstreamId = observation.Upstream.UpstreamId,
|
||||
DocumentVersion = observation.Upstream.DocumentVersion,
|
||||
FetchedAt = observation.Upstream.FetchedAt,
|
||||
ReceivedAt = observation.Upstream.ReceivedAt,
|
||||
ContentHash = observation.Upstream.ContentHash,
|
||||
Signature = new VexObservationSignatureRecord
|
||||
{
|
||||
Present = observation.Upstream.Signature.Present,
|
||||
Subject = observation.Upstream.Signature.Format,
|
||||
Issuer = observation.Upstream.Signature.KeyId,
|
||||
VerifiedAt = null
|
||||
}
|
||||
},
|
||||
Content = new VexObservationContentRecord
|
||||
{
|
||||
Format = observation.Content.Format,
|
||||
SpecVersion = observation.Content.SpecVersion,
|
||||
Raw = BsonDocument.Parse(observation.Content.Raw.ToJsonString())
|
||||
},
|
||||
Statements = observation.Statements.Select(ToStatementRecord).ToList(),
|
||||
Linkset = ToLinksetRecord(observation.Linkset),
|
||||
CreatedAt = observation.CreatedAt.UtcDateTime
|
||||
};
|
||||
}
|
||||
|
||||
private static VexObservationStatementRecord ToStatementRecord(VexObservationStatement statement)
|
||||
{
|
||||
return new VexObservationStatementRecord
|
||||
{
|
||||
VulnerabilityId = statement.VulnerabilityId,
|
||||
ProductKey = statement.ProductKey,
|
||||
Status = statement.Status.ToString().ToLowerInvariant(),
|
||||
LastObserved = statement.LastObserved,
|
||||
Locator = statement.Locator,
|
||||
Justification = statement.Justification?.ToString().ToLowerInvariant(),
|
||||
IntroducedVersion = statement.IntroducedVersion,
|
||||
FixedVersion = statement.FixedVersion,
|
||||
Detail = null,
|
||||
ScopeScore = null,
|
||||
Epss = null,
|
||||
Kev = null
|
||||
};
|
||||
}
|
||||
|
||||
private static VexObservationLinksetRecord ToLinksetRecord(VexObservationLinkset linkset)
|
||||
{
|
||||
return new VexObservationLinksetRecord
|
||||
{
|
||||
Aliases = linkset.Aliases.ToList(),
|
||||
Purls = linkset.Purls.ToList(),
|
||||
Cpes = linkset.Cpes.ToList(),
|
||||
References = linkset.References.Select(r => new VexObservationReferenceRecord
|
||||
{
|
||||
Type = r.Type,
|
||||
Url = r.Url
|
||||
}).ToList(),
|
||||
ReconciledFrom = linkset.ReconciledFrom.ToList(),
|
||||
Disagreements = linkset.Disagreements.Select(d => new VexLinksetDisagreementRecord
|
||||
{
|
||||
ProviderId = d.ProviderId,
|
||||
Status = d.Status,
|
||||
Justification = d.Justification,
|
||||
Confidence = d.Confidence
|
||||
}).ToList(),
|
||||
Observations = linkset.Observations.Select(o => new VexObservationLinksetObservationRecord
|
||||
{
|
||||
ObservationId = o.ObservationId,
|
||||
ProviderId = o.ProviderId,
|
||||
Status = o.Status,
|
||||
Confidence = o.Confidence
|
||||
}).ToList()
|
||||
};
|
||||
}
|
||||
|
||||
private static VexObservation ToModel(VexObservationRecord record)
|
||||
{
|
||||
var statements = record.Statements.Select(MapStatement).ToImmutableArray();
|
||||
var linkset = MapLinkset(record.Linkset);
|
||||
|
||||
var upstreamSignature = record.Upstream?.Signature is null
|
||||
? new VexObservationSignature(false, null, null, null)
|
||||
: new VexObservationSignature(
|
||||
record.Upstream.Signature.Present,
|
||||
record.Upstream.Signature.Subject,
|
||||
record.Upstream.Signature.Issuer,
|
||||
signature: null);
|
||||
|
||||
var upstream = record.Upstream is null
|
||||
? new VexObservationUpstream(
|
||||
upstreamId: record.ObservationId,
|
||||
documentVersion: null,
|
||||
fetchedAt: record.CreatedAt,
|
||||
receivedAt: record.CreatedAt,
|
||||
contentHash: record.Document.Digest,
|
||||
signature: upstreamSignature)
|
||||
: new VexObservationUpstream(
|
||||
record.Upstream.UpstreamId,
|
||||
record.Upstream.DocumentVersion,
|
||||
record.Upstream.FetchedAt,
|
||||
record.Upstream.ReceivedAt,
|
||||
record.Upstream.ContentHash,
|
||||
upstreamSignature);
|
||||
|
||||
var content = record.Content is null
|
||||
? new VexObservationContent("unknown", null, new JsonObject())
|
||||
: new VexObservationContent(
|
||||
record.Content.Format ?? "unknown",
|
||||
record.Content.SpecVersion,
|
||||
JsonNode.Parse(record.Content.Raw.ToJson()) ?? new JsonObject(),
|
||||
metadata: ImmutableDictionary<string, string>.Empty);
|
||||
|
||||
return new VexObservation(
|
||||
observationId: record.ObservationId,
|
||||
tenant: record.Tenant,
|
||||
providerId: record.ProviderId,
|
||||
streamId: string.IsNullOrWhiteSpace(record.StreamId) ? record.ProviderId : record.StreamId,
|
||||
upstream: upstream,
|
||||
statements: statements,
|
||||
content: content,
|
||||
linkset: linkset,
|
||||
createdAt: new DateTimeOffset(record.CreatedAt, TimeSpan.Zero),
|
||||
supersedes: ImmutableArray<string>.Empty,
|
||||
attributes: ImmutableDictionary<string, string>.Empty);
|
||||
}
|
||||
|
||||
private static VexObservationStatement MapStatement(VexObservationStatementRecord record)
|
||||
{
|
||||
var justification = string.IsNullOrWhiteSpace(record.Justification)
|
||||
? (VexJustification?)null
|
||||
: Enum.Parse<VexJustification>(record.Justification, ignoreCase: true);
|
||||
|
||||
return new VexObservationStatement(
|
||||
record.VulnerabilityId,
|
||||
record.ProductKey,
|
||||
Enum.Parse<VexClaimStatus>(record.Status, ignoreCase: true),
|
||||
record.LastObserved,
|
||||
locator: record.Locator,
|
||||
justification: justification,
|
||||
introducedVersion: record.IntroducedVersion,
|
||||
fixedVersion: record.FixedVersion,
|
||||
purl: null,
|
||||
cpe: null,
|
||||
evidence: null,
|
||||
metadata: ImmutableDictionary<string, string>.Empty);
|
||||
}
|
||||
|
||||
private static VexObservationLinkset MapLinkset(VexObservationLinksetRecord record)
|
||||
{
|
||||
var aliases = record?.Aliases?.Where(NotNullOrWhiteSpace).Select(a => a.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var purls = record?.Purls?.Where(NotNullOrWhiteSpace).Select(p => p.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var cpes = record?.Cpes?.Where(NotNullOrWhiteSpace).Select(c => c.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var references = record?.References?.Select(r => new VexObservationReference(r.Type, r.Url)).ToImmutableArray() ?? ImmutableArray<VexObservationReference>.Empty;
|
||||
var reconciledFrom = record?.ReconciledFrom?.Where(NotNullOrWhiteSpace).Select(r => r.Trim()).ToImmutableArray() ?? ImmutableArray<string>.Empty;
|
||||
var disagreements = record?.Disagreements?.Select(d => new VexObservationDisagreement(d.ProviderId, d.Status, d.Justification, d.Confidence)).ToImmutableArray() ?? ImmutableArray<VexObservationDisagreement>.Empty;
|
||||
var observationRefs = record?.Observations?.Select(o => new VexLinksetObservationRefModel(
|
||||
o.ObservationId,
|
||||
o.ProviderId,
|
||||
o.Status,
|
||||
o.Confidence)).ToImmutableArray() ?? ImmutableArray<VexLinksetObservationRefModel>.Empty;
|
||||
|
||||
return new VexObservationLinkset(aliases, purls, cpes, references, reconciledFrom, disagreements, observationRefs);
|
||||
}
|
||||
|
||||
private static bool NotNullOrWhiteSpace(string? value) => !string.IsNullOrWhiteSpace(value);
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
using System.Collections.Immutable;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB record for timeline events.
|
||||
/// </summary>
|
||||
[BsonIgnoreExtraElements]
|
||||
internal sealed class VexTimelineEventRecord
|
||||
{
|
||||
[BsonId]
|
||||
public string Id { get; set; } = default!;
|
||||
|
||||
public string Tenant { get; set; } = default!;
|
||||
|
||||
public string ProviderId { get; set; } = default!;
|
||||
|
||||
public string StreamId { get; set; } = default!;
|
||||
|
||||
public string EventType { get; set; } = default!;
|
||||
|
||||
public string TraceId { get; set; } = default!;
|
||||
|
||||
public string JustificationSummary { get; set; } = string.Empty;
|
||||
|
||||
public string? EvidenceHash { get; set; }
|
||||
|
||||
public string? PayloadHash { get; set; }
|
||||
|
||||
public DateTime CreatedAt { get; set; }
|
||||
= DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
|
||||
|
||||
public Dictionary<string, string> Attributes { get; set; } = new(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MongoDB implementation of the timeline event store.
|
||||
/// </summary>
|
||||
internal sealed class MongoVexTimelineEventStore : IVexTimelineEventStore
|
||||
{
|
||||
private readonly IMongoCollection<VexTimelineEventRecord> _collection;
|
||||
|
||||
public MongoVexTimelineEventStore(IMongoDatabase database)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
_collection = database.GetCollection<VexTimelineEventRecord>(VexMongoCollectionNames.TimelineEvents);
|
||||
}
|
||||
|
||||
public async ValueTask<string> InsertAsync(
|
||||
TimelineEvent evt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(evt);
|
||||
|
||||
var record = ToRecord(evt);
|
||||
|
||||
try
|
||||
{
|
||||
await _collection.InsertOneAsync(record, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return record.Id;
|
||||
}
|
||||
catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
|
||||
{
|
||||
// Event already exists, return the ID anyway
|
||||
return record.Id;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<int> InsertManyAsync(
|
||||
string tenant,
|
||||
IEnumerable<TimelineEvent> events,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var records = events
|
||||
.Where(e => e is not null && string.Equals(NormalizeTenant(e.Tenant), normalizedTenant, StringComparison.Ordinal))
|
||||
.Select(ToRecord)
|
||||
.ToList();
|
||||
|
||||
if (records.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var options = new InsertManyOptions { IsOrdered = false };
|
||||
try
|
||||
{
|
||||
await _collection.InsertManyAsync(records, options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return records.Count;
|
||||
}
|
||||
catch (MongoBulkWriteException<VexTimelineEventRecord> ex)
|
||||
{
|
||||
var duplicates = ex.WriteErrors?.Count(e => e.Category == ServerErrorCategory.DuplicateKey) ?? 0;
|
||||
return records.Count - duplicates;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByTimeRangeAsync(
|
||||
string tenant,
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var fromUtc = from.UtcDateTime;
|
||||
var toUtc = to.UtcDateTime;
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Gte(r => r.CreatedAt, fromUtc),
|
||||
Builders<VexTimelineEventRecord>.Filter.Lte(r => r.CreatedAt, toUtc));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Ascending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByTraceIdAsync(
|
||||
string tenant,
|
||||
string traceId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedTraceId = traceId?.Trim() ?? throw new ArgumentNullException(nameof(traceId));
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.TraceId, normalizedTraceId));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Ascending(r => r.CreatedAt))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByProviderAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedProvider = providerId?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(providerId));
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.ProviderId, normalizedProvider));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> FindByEventTypeAsync(
|
||||
string tenant,
|
||||
string eventType,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedType = eventType?.Trim().ToLowerInvariant()
|
||||
?? throw new ArgumentNullException(nameof(eventType));
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.EventType, normalizedType));
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<TimelineEvent>> GetRecentAsync(
|
||||
string tenant,
|
||||
int limit,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant);
|
||||
|
||||
var records = await _collection
|
||||
.Find(filter)
|
||||
.Sort(Builders<VexTimelineEventRecord>.Sort.Descending(r => r.CreatedAt))
|
||||
.Limit(Math.Max(1, limit))
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return records.Select(ToModel).ToList();
|
||||
}
|
||||
|
||||
public async ValueTask<TimelineEvent?> GetByIdAsync(
|
||||
string tenant,
|
||||
string eventId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var normalizedId = eventId?.Trim() ?? throw new ArgumentNullException(nameof(eventId));
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Id, normalizedId));
|
||||
|
||||
var record = await _collection
|
||||
.Find(filter)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return record is null ? null : ToModel(record);
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountAsync(
|
||||
string tenant,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant);
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask<long> CountInRangeAsync(
|
||||
string tenant,
|
||||
DateTimeOffset from,
|
||||
DateTimeOffset to,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedTenant = NormalizeTenant(tenant);
|
||||
var fromUtc = from.UtcDateTime;
|
||||
var toUtc = to.UtcDateTime;
|
||||
|
||||
var filter = Builders<VexTimelineEventRecord>.Filter.And(
|
||||
Builders<VexTimelineEventRecord>.Filter.Eq(r => r.Tenant, normalizedTenant),
|
||||
Builders<VexTimelineEventRecord>.Filter.Gte(r => r.CreatedAt, fromUtc),
|
||||
Builders<VexTimelineEventRecord>.Filter.Lte(r => r.CreatedAt, toUtc));
|
||||
|
||||
return await _collection
|
||||
.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string NormalizeTenant(string tenant)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tenant))
|
||||
{
|
||||
throw new ArgumentException("tenant is required", nameof(tenant));
|
||||
}
|
||||
|
||||
return tenant.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static VexTimelineEventRecord ToRecord(TimelineEvent evt)
|
||||
{
|
||||
return new VexTimelineEventRecord
|
||||
{
|
||||
Id = evt.EventId,
|
||||
Tenant = evt.Tenant,
|
||||
ProviderId = evt.ProviderId.ToLowerInvariant(),
|
||||
StreamId = evt.StreamId.ToLowerInvariant(),
|
||||
EventType = evt.EventType.ToLowerInvariant(),
|
||||
TraceId = evt.TraceId,
|
||||
JustificationSummary = evt.JustificationSummary,
|
||||
EvidenceHash = evt.EvidenceHash,
|
||||
PayloadHash = evt.PayloadHash,
|
||||
CreatedAt = evt.CreatedAt.UtcDateTime,
|
||||
Attributes = evt.Attributes.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal)
|
||||
};
|
||||
}
|
||||
|
||||
private static TimelineEvent ToModel(VexTimelineEventRecord record)
|
||||
{
|
||||
var attributes = record.Attributes?.ToImmutableDictionary(StringComparer.Ordinal)
|
||||
?? ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
return new TimelineEvent(
|
||||
eventId: record.Id,
|
||||
tenant: record.Tenant,
|
||||
providerId: record.ProviderId,
|
||||
streamId: record.StreamId,
|
||||
eventType: record.EventType,
|
||||
traceId: record.TraceId,
|
||||
justificationSummary: record.JustificationSummary,
|
||||
createdAt: new DateTimeOffset(DateTime.SpecifyKind(record.CreatedAt, DateTimeKind.Utc)),
|
||||
evidenceHash: record.EvidenceHash,
|
||||
payloadHash: record.PayloadHash,
|
||||
attributes: attributes);
|
||||
}
|
||||
}
|
||||
@@ -4,8 +4,8 @@ using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using StellaOps.Excititor.Storage.Mongo.Migrations;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
@@ -49,24 +49,32 @@ public static class VexMongoServiceCollectionExtensions
|
||||
|
||||
services.AddScoped<IVexRawStore, MongoVexRawStore>();
|
||||
services.AddScoped<IVexExportStore, MongoVexExportStore>();
|
||||
services.AddScoped<IVexProviderStore, MongoVexProviderStore>();
|
||||
services.AddScoped<IVexNormalizerRouter, StorageBackedVexNormalizerRouter>();
|
||||
services.AddScoped<IVexConsensusStore, MongoVexConsensusStore>();
|
||||
services.AddScoped<IVexConsensusHoldStore, MongoVexConsensusHoldStore>();
|
||||
services.AddScoped<IVexClaimStore, MongoVexClaimStore>();
|
||||
services.AddScoped<IVexCacheIndex, MongoVexCacheIndex>();
|
||||
services.AddScoped<IVexCacheMaintenance, MongoVexCacheMaintenance>();
|
||||
services.AddScoped<IVexConnectorStateRepository, MongoVexConnectorStateRepository>();
|
||||
services.AddScoped<IAirgapImportStore, MongoAirgapImportStore>();
|
||||
services.AddScoped<VexStatementBackfillService>();
|
||||
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
|
||||
services.AddSingleton<IVexMongoMigration, VexInitialIndexMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexRawSchemaMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexConsensusSignalsMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexConsensusHoldMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexObservationCollectionsMigration>();
|
||||
services.AddSingleton<VexMongoMigrationRunner>();
|
||||
services.AddHostedService<VexMongoMigrationHostedService>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
services.AddScoped<IVexProviderStore, MongoVexProviderStore>();
|
||||
services.AddScoped<IVexNormalizerRouter, StorageBackedVexNormalizerRouter>();
|
||||
services.AddScoped<IVexConsensusStore, MongoVexConsensusStore>();
|
||||
services.AddScoped<IVexConsensusHoldStore, MongoVexConsensusHoldStore>();
|
||||
services.AddScoped<IVexClaimStore, MongoVexClaimStore>();
|
||||
services.AddScoped<IVexCacheIndex, MongoVexCacheIndex>();
|
||||
services.AddScoped<IVexCacheMaintenance, MongoVexCacheMaintenance>();
|
||||
services.AddScoped<IVexConnectorStateRepository, MongoVexConnectorStateRepository>();
|
||||
services.AddScoped<IAirgapImportStore, MongoAirgapImportStore>();
|
||||
services.AddScoped<VexStatementBackfillService>();
|
||||
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
|
||||
services.AddScoped<IVexObservationStore, MongoVexObservationStore>();
|
||||
services.AddScoped<IVexLinksetStore, MongoVexLinksetStore>();
|
||||
services.AddScoped<IVexLinksetEventPublisher, MongoVexLinksetEventPublisher>();
|
||||
services.AddScoped<VexLinksetDisagreementService>();
|
||||
services.AddScoped<IVexTimelineEventStore, MongoVexTimelineEventStore>();
|
||||
services.AddScoped<IVexTimelineEventEmitter, VexTimelineEventEmitter>();
|
||||
services.AddSingleton<IVexMongoMigration, VexInitialIndexMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexTimelineEventIndexMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexRawSchemaMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexConsensusSignalsMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexConsensusHoldMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexObservationCollectionsMigration>();
|
||||
services.AddSingleton<IVexMongoMigration, VexRawIdempotencyIndexMigration>();
|
||||
services.AddSingleton<VexMongoMigrationRunner>();
|
||||
services.AddHostedService<VexMongoMigrationHostedService>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.RegularExpressions;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo.Validation;
|
||||
|
||||
/// <summary>
|
||||
/// Validates VEX raw documents against the schema defined in <see cref="Migrations.VexRawSchemaMigration"/>.
|
||||
/// Provides programmatic validation for operators to prove Excititor stores only immutable evidence.
|
||||
/// </summary>
|
||||
public static class VexRawSchemaValidator
|
||||
{
|
||||
private static readonly ImmutableHashSet<string> ValidFormats = ImmutableHashSet.Create(
|
||||
StringComparer.OrdinalIgnoreCase,
|
||||
"csaf", "cyclonedx", "openvex");
|
||||
|
||||
private static readonly ImmutableHashSet<BsonType> ValidContentTypes = ImmutableHashSet.Create(
|
||||
BsonType.Binary, BsonType.String);
|
||||
|
||||
private static readonly ImmutableHashSet<BsonType> ValidGridFsTypes = ImmutableHashSet.Create(
|
||||
BsonType.ObjectId, BsonType.Null, BsonType.String);
|
||||
|
||||
/// <summary>
|
||||
/// Validates a VEX raw document against the schema requirements.
|
||||
/// </summary>
|
||||
/// <param name="document">The document to validate.</param>
|
||||
/// <returns>Validation result with any violations found.</returns>
|
||||
public static VexRawValidationResult Validate(BsonDocument document)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var violations = new List<VexRawSchemaViolation>();
|
||||
|
||||
// Required fields
|
||||
ValidateRequired(document, "_id", violations);
|
||||
ValidateRequired(document, "providerId", violations);
|
||||
ValidateRequired(document, "format", violations);
|
||||
ValidateRequired(document, "sourceUri", violations);
|
||||
ValidateRequired(document, "retrievedAt", violations);
|
||||
ValidateRequired(document, "digest", violations);
|
||||
|
||||
// Field types and constraints
|
||||
ValidateStringField(document, "_id", minLength: 1, violations);
|
||||
ValidateStringField(document, "providerId", minLength: 1, violations);
|
||||
ValidateFormatEnum(document, violations);
|
||||
ValidateStringField(document, "sourceUri", minLength: 1, violations);
|
||||
ValidateDateField(document, "retrievedAt", violations);
|
||||
ValidateStringField(document, "digest", minLength: 32, violations);
|
||||
|
||||
// Optional fields with type constraints
|
||||
if (document.Contains("content"))
|
||||
{
|
||||
ValidateContentField(document, violations);
|
||||
}
|
||||
|
||||
if (document.Contains("gridFsObjectId"))
|
||||
{
|
||||
ValidateGridFsObjectIdField(document, violations);
|
||||
}
|
||||
|
||||
if (document.Contains("metadata"))
|
||||
{
|
||||
ValidateMetadataField(document, violations);
|
||||
}
|
||||
|
||||
return new VexRawValidationResult(
|
||||
document.GetValue("_id", BsonNull.Value).ToString() ?? "<unknown>",
|
||||
violations.Count == 0,
|
||||
violations.ToImmutableArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates multiple documents and returns aggregated results.
|
||||
/// </summary>
|
||||
public static VexRawBatchValidationResult ValidateBatch(IEnumerable<BsonDocument> documents)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(documents);
|
||||
|
||||
var results = new List<VexRawValidationResult>();
|
||||
foreach (var doc in documents)
|
||||
{
|
||||
results.Add(Validate(doc));
|
||||
}
|
||||
|
||||
var valid = results.Count(r => r.IsValid);
|
||||
var invalid = results.Count(r => !r.IsValid);
|
||||
|
||||
return new VexRawBatchValidationResult(
|
||||
results.Count,
|
||||
valid,
|
||||
invalid,
|
||||
results.Where(r => !r.IsValid).ToImmutableArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the MongoDB JSON Schema document for offline validation.
|
||||
/// </summary>
|
||||
public static BsonDocument GetJsonSchema()
|
||||
{
|
||||
var properties = new BsonDocument
|
||||
{
|
||||
{ "_id", new BsonDocument { { "bsonType", "string" }, { "description", "Content digest serving as immutable key" } } },
|
||||
{ "providerId", new BsonDocument { { "bsonType", "string" }, { "minLength", 1 }, { "description", "VEX provider identifier" } } },
|
||||
{ "format", new BsonDocument
|
||||
{
|
||||
{ "bsonType", "string" },
|
||||
{ "enum", new BsonArray { "csaf", "cyclonedx", "openvex" } },
|
||||
{ "description", "VEX document format" }
|
||||
}
|
||||
},
|
||||
{ "sourceUri", new BsonDocument { { "bsonType", "string" }, { "minLength", 1 }, { "description", "Original source URI" } } },
|
||||
{ "retrievedAt", new BsonDocument { { "bsonType", "date" }, { "description", "Timestamp when document was fetched" } } },
|
||||
{ "digest", new BsonDocument { { "bsonType", "string" }, { "minLength", 32 }, { "description", "Content hash (SHA-256 hex)" } } },
|
||||
{ "content", new BsonDocument
|
||||
{
|
||||
{ "bsonType", new BsonArray { "binData", "string" } },
|
||||
{ "description", "Raw document content (binary or base64 string)" }
|
||||
}
|
||||
},
|
||||
{ "gridFsObjectId", new BsonDocument
|
||||
{
|
||||
{ "bsonType", new BsonArray { "objectId", "null", "string" } },
|
||||
{ "description", "GridFS reference for large documents" }
|
||||
}
|
||||
},
|
||||
{ "metadata", new BsonDocument
|
||||
{
|
||||
{ "bsonType", "object" },
|
||||
{ "additionalProperties", true },
|
||||
{ "description", "Provider-specific metadata (string values only)" }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return new BsonDocument
|
||||
{
|
||||
{
|
||||
"$jsonSchema",
|
||||
new BsonDocument
|
||||
{
|
||||
{ "bsonType", "object" },
|
||||
{ "title", "VEX Raw Document Schema" },
|
||||
{ "description", "Schema for immutable VEX evidence storage. Documents are content-addressed and must not be modified after insertion." },
|
||||
{ "required", new BsonArray { "_id", "providerId", "format", "sourceUri", "retrievedAt", "digest" } },
|
||||
{ "properties", properties },
|
||||
{ "additionalProperties", true }
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the schema as a JSON string for operator documentation.
|
||||
/// </summary>
|
||||
public static string GetJsonSchemaAsJson()
|
||||
{
|
||||
return GetJsonSchema().ToJson(new MongoDB.Bson.IO.JsonWriterSettings { Indent = true });
|
||||
}
|
||||
|
||||
private static void ValidateRequired(BsonDocument doc, string field, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
if (!doc.Contains(field) || doc[field].IsBsonNull)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation(field, $"Required field '{field}' is missing or null"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateStringField(BsonDocument doc, string field, int minLength, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
if (!doc.Contains(field))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var value = doc[field];
|
||||
if (value.IsBsonNull)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!value.IsString)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation(field, $"Field '{field}' must be a string, got {value.BsonType}"));
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.AsString.Length < minLength)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation(field, $"Field '{field}' must have minimum length {minLength}, got {value.AsString.Length}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateFormatEnum(BsonDocument doc, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
if (!doc.Contains("format"))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var value = doc["format"];
|
||||
if (value.IsBsonNull || !value.IsString)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ValidFormats.Contains(value.AsString))
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation("format", $"Field 'format' must be one of [{string.Join(", ", ValidFormats)}], got '{value.AsString}'"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateDateField(BsonDocument doc, string field, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
if (!doc.Contains(field))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var value = doc[field];
|
||||
if (value.IsBsonNull)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.BsonType != BsonType.DateTime)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation(field, $"Field '{field}' must be a date, got {value.BsonType}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateContentField(BsonDocument doc, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
var value = doc["content"];
|
||||
if (value.IsBsonNull)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ValidContentTypes.Contains(value.BsonType))
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation("content", $"Field 'content' must be binary or string, got {value.BsonType}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateGridFsObjectIdField(BsonDocument doc, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
var value = doc["gridFsObjectId"];
|
||||
if (!ValidGridFsTypes.Contains(value.BsonType))
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation("gridFsObjectId", $"Field 'gridFsObjectId' must be objectId, null, or string, got {value.BsonType}"));
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateMetadataField(BsonDocument doc, List<VexRawSchemaViolation> violations)
|
||||
{
|
||||
var value = doc["metadata"];
|
||||
if (value.IsBsonNull)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (value.BsonType != BsonType.Document)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation("metadata", $"Field 'metadata' must be an object, got {value.BsonType}"));
|
||||
return;
|
||||
}
|
||||
|
||||
var metadata = value.AsBsonDocument;
|
||||
foreach (var element in metadata)
|
||||
{
|
||||
if (!element.Value.IsString && !element.Value.IsBsonNull)
|
||||
{
|
||||
violations.Add(new VexRawSchemaViolation($"metadata.{element.Name}", $"Metadata field '{element.Name}' must be a string, got {element.Value.BsonType}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a schema violation found during validation.
|
||||
/// </summary>
|
||||
public sealed record VexRawSchemaViolation(string Field, string Message);
|
||||
|
||||
/// <summary>
|
||||
/// Result of validating a single VEX raw document.
|
||||
/// </summary>
|
||||
public sealed record VexRawValidationResult(
|
||||
string DocumentId,
|
||||
bool IsValid,
|
||||
ImmutableArray<VexRawSchemaViolation> Violations);
|
||||
|
||||
/// <summary>
|
||||
/// Result of validating a batch of VEX raw documents.
|
||||
/// </summary>
|
||||
public sealed record VexRawBatchValidationResult(
|
||||
int TotalCount,
|
||||
int ValidCount,
|
||||
int InvalidCount,
|
||||
ImmutableArray<VexRawValidationResult> InvalidDocuments);
|
||||
@@ -47,6 +47,7 @@ public static class VexMongoMappingRegistry
|
||||
RegisterClassMap<VexConnectorStateDocument>();
|
||||
RegisterClassMap<VexConsensusHoldRecord>();
|
||||
RegisterClassMap<AirgapImportRecord>();
|
||||
RegisterClassMap<VexTimelineEventRecord>();
|
||||
}
|
||||
|
||||
private static void RegisterClassMap<TDocument>()
|
||||
@@ -80,5 +81,7 @@ public static class VexMongoCollectionNames
|
||||
public const string Attestations = "vex.attestations";
|
||||
public const string Observations = "vex.observations";
|
||||
public const string Linksets = "vex.linksets";
|
||||
public const string LinksetEvents = "vex.linkset_events";
|
||||
public const string AirgapImports = "vex.airgap_imports";
|
||||
public const string TimelineEvents = "vex.timeline_events";
|
||||
}
|
||||
|
||||
@@ -490,6 +490,10 @@ internal sealed class VexLinksetRecord
|
||||
|
||||
public DateTime CreatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
|
||||
|
||||
public DateTime UpdatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
|
||||
|
||||
public List<VexObservationLinksetObservationRecord> Observations { get; set; } = new();
|
||||
|
||||
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
|
||||
}
|
||||
|
||||
@@ -1310,6 +1314,21 @@ internal sealed class VexConnectorStateDocument
|
||||
public string? LastFailureReason { get; set; }
|
||||
= null;
|
||||
|
||||
public DateTime? LastHeartbeatAt { get; set; }
|
||||
= null;
|
||||
|
||||
public string? LastHeartbeatStatus { get; set; }
|
||||
= null;
|
||||
|
||||
public string? LastArtifactHash { get; set; }
|
||||
= null;
|
||||
|
||||
public string? LastArtifactKind { get; set; }
|
||||
= null;
|
||||
|
||||
public string? LastCheckpoint { get; set; }
|
||||
= null;
|
||||
|
||||
public static VexConnectorStateDocument FromRecord(VexConnectorState state)
|
||||
=> new()
|
||||
{
|
||||
@@ -1323,6 +1342,11 @@ internal sealed class VexConnectorStateDocument
|
||||
FailureCount = state.FailureCount,
|
||||
NextEligibleRun = state.NextEligibleRun?.UtcDateTime,
|
||||
LastFailureReason = state.LastFailureReason,
|
||||
LastHeartbeatAt = state.LastHeartbeatAt?.UtcDateTime,
|
||||
LastHeartbeatStatus = state.LastHeartbeatStatus,
|
||||
LastArtifactHash = state.LastArtifactHash,
|
||||
LastArtifactKind = state.LastArtifactKind,
|
||||
LastCheckpoint = state.LastCheckpoint,
|
||||
};
|
||||
|
||||
public VexConnectorState ToRecord()
|
||||
@@ -1336,6 +1360,9 @@ internal sealed class VexConnectorStateDocument
|
||||
var nextEligibleRun = NextEligibleRun.HasValue
|
||||
? new DateTimeOffset(DateTime.SpecifyKind(NextEligibleRun.Value, DateTimeKind.Utc))
|
||||
: (DateTimeOffset?)null;
|
||||
var lastHeartbeatAt = LastHeartbeatAt.HasValue
|
||||
? new DateTimeOffset(DateTime.SpecifyKind(LastHeartbeatAt.Value, DateTimeKind.Utc))
|
||||
: (DateTimeOffset?)null;
|
||||
|
||||
return new VexConnectorState(
|
||||
ConnectorId,
|
||||
@@ -1345,6 +1372,52 @@ internal sealed class VexConnectorStateDocument
|
||||
lastSuccessAt,
|
||||
FailureCount,
|
||||
nextEligibleRun,
|
||||
string.IsNullOrWhiteSpace(LastFailureReason) ? null : LastFailureReason);
|
||||
string.IsNullOrWhiteSpace(LastFailureReason) ? null : LastFailureReason,
|
||||
lastHeartbeatAt,
|
||||
LastHeartbeatStatus,
|
||||
LastArtifactHash,
|
||||
LastArtifactKind,
|
||||
LastCheckpoint);
|
||||
}
|
||||
}
|
||||
|
||||
[BsonIgnoreExtraElements]
|
||||
internal sealed class VexLinksetEventRecord
|
||||
{
|
||||
[BsonId]
|
||||
public string Id { get; set; } = default!;
|
||||
|
||||
public string EventType { get; set; } = default!;
|
||||
|
||||
public string Tenant { get; set; } = default!;
|
||||
|
||||
public string LinksetId { get; set; } = default!;
|
||||
|
||||
public string VulnerabilityId { get; set; } = default!;
|
||||
|
||||
public string ProductKey { get; set; } = default!;
|
||||
|
||||
public List<VexLinksetEventObservationRecord> Observations { get; set; } = new();
|
||||
|
||||
public List<VexLinksetDisagreementRecord> Disagreements { get; set; } = new();
|
||||
|
||||
public DateTime CreatedAtUtc { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
|
||||
|
||||
public DateTime PublishedAtUtc { get; set; } = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc);
|
||||
|
||||
public int ConflictCount { get; set; } = 0;
|
||||
|
||||
public int ObservationCount { get; set; } = 0;
|
||||
}
|
||||
|
||||
[BsonIgnoreExtraElements]
|
||||
internal sealed class VexLinksetEventObservationRecord
|
||||
{
|
||||
public string ObservationId { get; set; } = default!;
|
||||
|
||||
public string ProviderId { get; set; } = default!;
|
||||
|
||||
public string Status { get; set; } = default!;
|
||||
|
||||
public double? Confidence { get; set; } = null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
|
||||
namespace StellaOps.Excititor.Storage.Mongo;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of <see cref="IVexTimelineEventEmitter"/> that persists events to MongoDB.
|
||||
/// </summary>
|
||||
internal sealed class VexTimelineEventEmitter : IVexTimelineEventEmitter
|
||||
{
|
||||
private readonly IVexTimelineEventStore _store;
|
||||
private readonly ILogger<VexTimelineEventEmitter> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public VexTimelineEventEmitter(
|
||||
IVexTimelineEventStore store,
|
||||
ILogger<VexTimelineEventEmitter> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_store = store ?? throw new ArgumentNullException(nameof(store));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async ValueTask EmitObservationIngestAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
string streamId,
|
||||
string traceId,
|
||||
string observationId,
|
||||
string evidenceHash,
|
||||
string justificationSummary,
|
||||
ImmutableDictionary<string, string>? attributes = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var eventAttributes = (attributes ?? ImmutableDictionary<string, string>.Empty)
|
||||
.SetItem(VexTimelineEventAttributes.ObservationId, observationId);
|
||||
|
||||
var evt = new TimelineEvent(
|
||||
eventId: GenerateEventId(tenant, providerId, VexTimelineEventTypes.ObservationIngested),
|
||||
tenant: tenant,
|
||||
providerId: providerId,
|
||||
streamId: streamId,
|
||||
eventType: VexTimelineEventTypes.ObservationIngested,
|
||||
traceId: traceId,
|
||||
justificationSummary: justificationSummary,
|
||||
createdAt: _timeProvider.GetUtcNow(),
|
||||
evidenceHash: evidenceHash,
|
||||
payloadHash: null,
|
||||
attributes: eventAttributes);
|
||||
|
||||
await EmitAsync(evt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask EmitLinksetUpdateAsync(
|
||||
string tenant,
|
||||
string providerId,
|
||||
string streamId,
|
||||
string traceId,
|
||||
string linksetId,
|
||||
string vulnerabilityId,
|
||||
string productKey,
|
||||
string payloadHash,
|
||||
string justificationSummary,
|
||||
ImmutableDictionary<string, string>? attributes = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var eventAttributes = (attributes ?? ImmutableDictionary<string, string>.Empty)
|
||||
.SetItem(VexTimelineEventAttributes.LinksetId, linksetId)
|
||||
.SetItem(VexTimelineEventAttributes.VulnerabilityId, vulnerabilityId)
|
||||
.SetItem(VexTimelineEventAttributes.ProductKey, productKey);
|
||||
|
||||
var evt = new TimelineEvent(
|
||||
eventId: GenerateEventId(tenant, providerId, VexTimelineEventTypes.LinksetUpdated),
|
||||
tenant: tenant,
|
||||
providerId: providerId,
|
||||
streamId: streamId,
|
||||
eventType: VexTimelineEventTypes.LinksetUpdated,
|
||||
traceId: traceId,
|
||||
justificationSummary: justificationSummary,
|
||||
createdAt: _timeProvider.GetUtcNow(),
|
||||
evidenceHash: null,
|
||||
payloadHash: payloadHash,
|
||||
attributes: eventAttributes);
|
||||
|
||||
await EmitAsync(evt, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async ValueTask EmitAsync(
|
||||
TimelineEvent evt,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(evt);
|
||||
|
||||
try
|
||||
{
|
||||
var eventId = await _store.InsertAsync(evt, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Timeline event emitted: {EventType} for tenant {Tenant}, provider {ProviderId}, trace {TraceId}",
|
||||
evt.EventType,
|
||||
evt.Tenant,
|
||||
evt.ProviderId,
|
||||
evt.TraceId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to emit timeline event {EventType} for tenant {Tenant}, provider {ProviderId}: {Message}",
|
||||
evt.EventType,
|
||||
evt.Tenant,
|
||||
evt.ProviderId,
|
||||
ex.Message);
|
||||
|
||||
// Don't throw - timeline events are non-critical and shouldn't block main operations
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask EmitBatchAsync(
|
||||
string tenant,
|
||||
IEnumerable<TimelineEvent> events,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(events);
|
||||
|
||||
var eventList = events.ToList();
|
||||
if (eventList.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var insertedCount = await _store.InsertManyAsync(tenant, eventList, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Batch timeline events emitted: {InsertedCount}/{TotalCount} for tenant {Tenant}",
|
||||
insertedCount,
|
||||
eventList.Count,
|
||||
tenant);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to emit batch timeline events for tenant {Tenant}: {Message}",
|
||||
tenant,
|
||||
ex.Message);
|
||||
|
||||
// Don't throw - timeline events are non-critical
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a deterministic event ID based on tenant, provider, event type, and timestamp.
|
||||
/// </summary>
|
||||
private string GenerateEventId(string tenant, string providerId, string eventType)
|
||||
{
|
||||
var timestamp = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds();
|
||||
var input = $"{tenant}|{providerId}|{eventType}|{timestamp}|{Guid.NewGuid():N}";
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input));
|
||||
return $"evt:{Convert.ToHexString(hash).ToLowerInvariant()[..32]}";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user