save work
This commit is contained in:
@@ -19,6 +19,7 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\\..\\__Libraries\\StellaOps.Plugin\\StellaOps.Plugin.csproj" />
|
||||
<ProjectReference Include="..\\__Libraries\\StellaOps.Scanner.Surface.FS\\StellaOps.Scanner.Surface.FS.csproj" />
|
||||
<ProjectReference Include="..\\__Libraries\\StellaOps.Scanner.Surface.Secrets\\StellaOps.Scanner.Surface.Secrets.csproj" />
|
||||
<ProjectReference Include="..\\__Libraries\\StellaOps.Scanner.Surface.Env\\StellaOps.Scanner.Surface.Env.csproj" />
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using StellaOps.Scanner.Core;
|
||||
using StellaOps.Scanner.WebService.Contracts;
|
||||
using StellaOps.Scanner.WebService.Services;
|
||||
|
||||
@@ -85,7 +86,7 @@ internal static class ScoreReplayEndpoints
|
||||
RootHash: result.RootHash,
|
||||
BundleUri: result.BundleUri,
|
||||
ManifestHash: result.ManifestHash,
|
||||
ReplayedAtUtc: result.ReplayedAt,
|
||||
ReplayedAt: result.ReplayedAt,
|
||||
Deterministic: result.Deterministic));
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
@@ -107,6 +108,8 @@ internal static class ScoreReplayEndpoints
|
||||
string scanId,
|
||||
[FromQuery] string? rootHash,
|
||||
IScoreReplayService replayService,
|
||||
IProofBundleWriter bundleWriter,
|
||||
IScanManifestSigner manifestSigner,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(scanId))
|
||||
@@ -131,11 +134,29 @@ internal static class ScoreReplayEndpoints
|
||||
});
|
||||
}
|
||||
|
||||
bool manifestDsseValid;
|
||||
try
|
||||
{
|
||||
var contents = await bundleWriter.ReadBundleAsync(bundle.BundleUri, cancellationToken).ConfigureAwait(false);
|
||||
var verify = await manifestSigner.VerifyAsync(contents.SignedManifest, cancellationToken).ConfigureAwait(false);
|
||||
manifestDsseValid = verify.IsValid;
|
||||
}
|
||||
catch (FileNotFoundException ex)
|
||||
{
|
||||
return Results.NotFound(new ProblemDetails
|
||||
{
|
||||
Title = "Bundle not found",
|
||||
Detail = ex.Message,
|
||||
Status = StatusCodes.Status404NotFound
|
||||
});
|
||||
}
|
||||
|
||||
return Results.Ok(new ScoreBundleResponse(
|
||||
ScanId: bundle.ScanId,
|
||||
RootHash: bundle.RootHash,
|
||||
BundleUri: bundle.BundleUri,
|
||||
CreatedAtUtc: bundle.CreatedAtUtc));
|
||||
ManifestDsseValid: manifestDsseValid,
|
||||
CreatedAt: bundle.CreatedAtUtc));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -213,14 +234,14 @@ public sealed record ScoreReplayRequest(
|
||||
/// <param name="RootHash">Root hash of the proof ledger.</param>
|
||||
/// <param name="BundleUri">URI to the proof bundle.</param>
|
||||
/// <param name="ManifestHash">Hash of the manifest used.</param>
|
||||
/// <param name="ReplayedAtUtc">When the replay was performed.</param>
|
||||
/// <param name="ReplayedAt">When the replay was performed.</param>
|
||||
/// <param name="Deterministic">Whether the replay was deterministic.</param>
|
||||
public sealed record ScoreReplayResponse(
|
||||
double Score,
|
||||
string RootHash,
|
||||
string BundleUri,
|
||||
string ManifestHash,
|
||||
DateTimeOffset ReplayedAtUtc,
|
||||
DateTimeOffset ReplayedAt,
|
||||
bool Deterministic);
|
||||
|
||||
/// <summary>
|
||||
@@ -230,7 +251,8 @@ public sealed record ScoreBundleResponse(
|
||||
string ScanId,
|
||||
string RootHash,
|
||||
string BundleUri,
|
||||
DateTimeOffset CreatedAtUtc);
|
||||
bool ManifestDsseValid,
|
||||
DateTimeOffset CreatedAt);
|
||||
|
||||
/// <summary>
|
||||
/// Request for bundle verification.
|
||||
|
||||
@@ -92,6 +92,11 @@ public sealed class ScannerWebServiceOptions
|
||||
/// </summary>
|
||||
public DeterminismOptions Determinism { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Score replay configuration (disabled by default).
|
||||
/// </summary>
|
||||
public ScoreReplayOptions ScoreReplay { get; set; } = new();
|
||||
|
||||
public sealed class StorageOptions
|
||||
{
|
||||
public string Driver { get; set; } = "postgres";
|
||||
@@ -440,4 +445,19 @@ public sealed class ScannerWebServiceOptions
|
||||
|
||||
public string? PolicySnapshotId { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ScoreReplayOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Enables score replay endpoints (/api/v1/score/*).
|
||||
/// Default: false.
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Directory used to persist proof bundles created during replay.
|
||||
/// When empty, the host selects a safe default (tests use temp storage).
|
||||
/// </summary>
|
||||
public string BundleStoragePath { get; set; } = string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ using StellaOps.Cryptography.Plugin.BouncyCastle;
|
||||
using StellaOps.Concelier.Core.Linksets;
|
||||
using StellaOps.Policy;
|
||||
using StellaOps.Scanner.Cache;
|
||||
using StellaOps.Scanner.Core;
|
||||
using StellaOps.Scanner.Core.Configuration;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Core.TrustAnchors;
|
||||
@@ -124,6 +125,27 @@ builder.Services.AddSingleton<IPolicyAuditRepository, InMemoryPolicyAuditReposit
|
||||
builder.Services.AddSingleton<PolicySnapshotStore>();
|
||||
builder.Services.AddSingleton<PolicyPreviewService>();
|
||||
builder.Services.AddSingleton<IRecordModeService, RecordModeService>();
|
||||
builder.Services.AddSingleton<IScoreReplayService, ScoreReplayService>();
|
||||
builder.Services.AddSingleton<IScanManifestRepository, InMemoryScanManifestRepository>();
|
||||
builder.Services.AddSingleton<IProofBundleRepository, InMemoryProofBundleRepository>();
|
||||
builder.Services.AddSingleton<IScoringService, DeterministicScoringService>();
|
||||
builder.Services.AddSingleton<IScanManifestSigner, ScanManifestSigner>();
|
||||
builder.Services.AddSingleton<IProofBundleWriter>(sp =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<ScannerWebServiceOptions>>().Value;
|
||||
var hostEnvironment = sp.GetRequiredService<IHostEnvironment>();
|
||||
|
||||
var configuredPath = options.ScoreReplay.BundleStoragePath?.Trim() ?? string.Empty;
|
||||
var defaultPath = hostEnvironment.IsEnvironment("Testing")
|
||||
? Path.Combine(Path.GetTempPath(), "stellaops-proofs-testing")
|
||||
: Path.Combine(Path.GetTempPath(), "stellaops-proofs");
|
||||
|
||||
return new ProofBundleWriter(new ProofBundleWriterOptions
|
||||
{
|
||||
StorageBasePath = string.IsNullOrWhiteSpace(configuredPath) ? defaultPath : configuredPath,
|
||||
ContentAddressed = true
|
||||
});
|
||||
});
|
||||
builder.Services.AddReachabilityDrift();
|
||||
builder.Services.AddStellaOpsCrypto();
|
||||
builder.Services.AddBouncyCastleEd25519Provider();
|
||||
@@ -470,7 +492,12 @@ apiGroup.MapScanEndpoints(resolvedOptions.Api.ScansSegment);
|
||||
apiGroup.MapReachabilityDriftRootEndpoints();
|
||||
apiGroup.MapProofSpineEndpoints(resolvedOptions.Api.SpinesSegment, resolvedOptions.Api.ScansSegment);
|
||||
apiGroup.MapReplayEndpoints();
|
||||
if (resolvedOptions.ScoreReplay.Enabled)
|
||||
{
|
||||
apiGroup.MapScoreReplayEndpoints();
|
||||
}
|
||||
apiGroup.MapWitnessEndpoints(); // Sprint: SPRINT_3700_0001_0001
|
||||
apiGroup.MapEpssEndpoints(); // Sprint: SPRINT_3410_0002_0001
|
||||
|
||||
if (resolvedOptions.Features.EnablePolicyPreview)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using StellaOps.Policy.Scoring;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Services;
|
||||
|
||||
public sealed class DeterministicScoringService : IScoringService
|
||||
{
|
||||
public Task<double> ReplayScoreAsync(
|
||||
string scanId,
|
||||
string concelierSnapshotHash,
|
||||
string excititorSnapshotHash,
|
||||
string latticePolicyHash,
|
||||
byte[] seed,
|
||||
DateTimeOffset freezeTimestamp,
|
||||
ProofLedger ledger,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(scanId);
|
||||
ArgumentNullException.ThrowIfNull(seed);
|
||||
ArgumentNullException.ThrowIfNull(ledger);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var input = string.Join(
|
||||
"|",
|
||||
scanId.Trim(),
|
||||
concelierSnapshotHash?.Trim() ?? string.Empty,
|
||||
excititorSnapshotHash?.Trim() ?? string.Empty,
|
||||
latticePolicyHash?.Trim() ?? string.Empty,
|
||||
freezeTimestamp.ToUniversalTime().ToString("O"),
|
||||
Convert.ToHexStringLower(seed));
|
||||
|
||||
var digest = SHA256.HashData(Encoding.UTF8.GetBytes(input));
|
||||
var value = BinaryPrimitives.ReadUInt64BigEndian(digest.AsSpan(0, sizeof(ulong)));
|
||||
var score = value / (double)ulong.MaxValue;
|
||||
score = Math.Clamp(score, 0.0, 1.0);
|
||||
|
||||
var actor = "scanner.webservice.score";
|
||||
var evidenceRefs = new[]
|
||||
{
|
||||
concelierSnapshotHash,
|
||||
excititorSnapshotHash,
|
||||
latticePolicyHash
|
||||
}.Where(v => !string.IsNullOrWhiteSpace(v)).ToArray();
|
||||
|
||||
var inputNodeId = $"input:{scanId}";
|
||||
ledger.Append(ProofNode.CreateInput(
|
||||
id: inputNodeId,
|
||||
ruleId: "deterministic",
|
||||
actor: actor,
|
||||
tsUtc: freezeTimestamp,
|
||||
seed: seed,
|
||||
initialValue: score,
|
||||
evidenceRefs: evidenceRefs));
|
||||
|
||||
ledger.Append(ProofNode.CreateScore(
|
||||
id: $"score:{scanId}",
|
||||
ruleId: "deterministic",
|
||||
actor: actor,
|
||||
tsUtc: freezeTimestamp,
|
||||
seed: seed,
|
||||
finalScore: score,
|
||||
parentIds: new[] { inputNodeId }));
|
||||
|
||||
return Task.FromResult(score);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
using System.Collections.Concurrent;
|
||||
using StellaOps.Scanner.Core;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Services;
|
||||
|
||||
public sealed class InMemoryProofBundleRepository : IProofBundleRepository
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, ProofBundle>> _bundles
|
||||
= new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public Task<ProofBundle?> GetBundleAsync(string scanId, string? rootHash = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(scanId))
|
||||
{
|
||||
return Task.FromResult<ProofBundle?>(null);
|
||||
}
|
||||
|
||||
if (!_bundles.TryGetValue(scanId.Trim(), out var bundlesByRootHash) || bundlesByRootHash.Count == 0)
|
||||
{
|
||||
return Task.FromResult<ProofBundle?>(null);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(rootHash))
|
||||
{
|
||||
var normalizedHash = NormalizeDigest(rootHash);
|
||||
return Task.FromResult(bundlesByRootHash.TryGetValue(normalizedHash, out var bundle) ? bundle : null);
|
||||
}
|
||||
|
||||
var best = bundlesByRootHash.Values
|
||||
.OrderByDescending(b => b.CreatedAtUtc)
|
||||
.ThenBy(b => b.RootHash, StringComparer.Ordinal)
|
||||
.FirstOrDefault();
|
||||
|
||||
return Task.FromResult(best);
|
||||
}
|
||||
|
||||
public Task SaveBundleAsync(ProofBundle bundle, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(bundle);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var bundlesByRootHash = _bundles.GetOrAdd(
|
||||
bundle.ScanId.Trim(),
|
||||
_ => new ConcurrentDictionary<string, ProofBundle>(StringComparer.OrdinalIgnoreCase));
|
||||
|
||||
bundlesByRootHash[NormalizeDigest(bundle.RootHash)] = bundle;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static string NormalizeDigest(string value)
|
||||
{
|
||||
var trimmed = value.Trim();
|
||||
if (!trimmed.Contains(':', StringComparison.Ordinal))
|
||||
{
|
||||
trimmed = $"sha256:{trimmed}";
|
||||
}
|
||||
|
||||
return trimmed.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Scanner.Core;
|
||||
using StellaOps.Scanner.WebService.Domain;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Services;
|
||||
|
||||
public sealed class InMemoryScanManifestRepository : IScanManifestRepository
|
||||
{
|
||||
private readonly IScanCoordinator _scanCoordinator;
|
||||
private readonly IScanManifestSigner _manifestSigner;
|
||||
private readonly ILogger<InMemoryScanManifestRepository> _logger;
|
||||
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, SignedScanManifest>> _manifestsByScanId
|
||||
= new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public InMemoryScanManifestRepository(
|
||||
IScanCoordinator scanCoordinator,
|
||||
IScanManifestSigner manifestSigner,
|
||||
ILogger<InMemoryScanManifestRepository> logger)
|
||||
{
|
||||
_scanCoordinator = scanCoordinator ?? throw new ArgumentNullException(nameof(scanCoordinator));
|
||||
_manifestSigner = manifestSigner ?? throw new ArgumentNullException(nameof(manifestSigner));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<SignedScanManifest?> GetManifestAsync(
|
||||
string scanId,
|
||||
string? manifestHash = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(scanId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalizedScanId = scanId.Trim();
|
||||
var normalizedManifestHash = NormalizeDigest(manifestHash);
|
||||
|
||||
if (_manifestsByScanId.TryGetValue(normalizedScanId, out var existingByHash))
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(normalizedManifestHash))
|
||||
{
|
||||
return existingByHash.TryGetValue(normalizedManifestHash, out var hit) ? hit : null;
|
||||
}
|
||||
|
||||
return SelectDefault(existingByHash);
|
||||
}
|
||||
|
||||
var snapshot = await _scanCoordinator.GetAsync(new ScanId(normalizedScanId), cancellationToken).ConfigureAwait(false);
|
||||
if (snapshot is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var manifest = BuildManifest(snapshot);
|
||||
var signed = await _manifestSigner.SignAsync(manifest, cancellationToken).ConfigureAwait(false);
|
||||
await SaveManifestAsync(signed, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(normalizedManifestHash)
|
||||
&& !string.Equals(NormalizeDigest(signed.ManifestHash), normalizedManifestHash, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
_logger.LogInformation("Created scan manifest for scan {ScanId} ({ManifestHash})", normalizedScanId, signed.ManifestHash);
|
||||
return signed;
|
||||
}
|
||||
|
||||
public Task SaveManifestAsync(SignedScanManifest manifest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var scanId = manifest.Manifest.ScanId.Trim();
|
||||
var hash = NormalizeDigest(manifest.ManifestHash) ?? manifest.ManifestHash.Trim();
|
||||
|
||||
var byHash = _manifestsByScanId.GetOrAdd(scanId, _ => new ConcurrentDictionary<string, SignedScanManifest>(StringComparer.OrdinalIgnoreCase));
|
||||
byHash[hash] = manifest;
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<List<string>> FindAffectedScansAsync(AffectedScansQuery query, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(query);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
return Task.FromResult(new List<string>());
|
||||
}
|
||||
|
||||
private static SignedScanManifest? SelectDefault(ConcurrentDictionary<string, SignedScanManifest> manifestsByHash)
|
||||
{
|
||||
if (manifestsByHash.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return manifestsByHash.Values
|
||||
.OrderByDescending(m => m.Manifest.CreatedAtUtc)
|
||||
.ThenBy(m => m.ManifestHash, StringComparer.Ordinal)
|
||||
.FirstOrDefault();
|
||||
}
|
||||
|
||||
private static ScanManifest BuildManifest(ScanSnapshot snapshot)
|
||||
{
|
||||
var targetDigest = NormalizeDigest(snapshot.Target.Digest) ?? "sha256:unknown";
|
||||
var seed = SHA256.HashData(Encoding.UTF8.GetBytes(snapshot.ScanId.Value));
|
||||
|
||||
var version = typeof(InMemoryScanManifestRepository).Assembly.GetName().Version?.ToString() ?? "0.0.0";
|
||||
|
||||
return ScanManifest.CreateBuilder(snapshot.ScanId.Value, targetDigest)
|
||||
.WithCreatedAt(snapshot.CreatedAt)
|
||||
.WithScannerVersion(version)
|
||||
.WithWorkerVersion(version)
|
||||
.WithConcelierSnapshot(ComputeDigest($"concelier:{snapshot.ScanId.Value}"))
|
||||
.WithExcititorSnapshot(ComputeDigest($"excititor:{snapshot.ScanId.Value}"))
|
||||
.WithLatticePolicyHash(ComputeDigest($"policy:{snapshot.ScanId.Value}"))
|
||||
.WithDeterministic(true)
|
||||
.WithSeed(seed)
|
||||
.Build();
|
||||
}
|
||||
|
||||
private static string ComputeDigest(string value)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(value);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return $"sha256:{Convert.ToHexStringLower(hash)}";
|
||||
}
|
||||
|
||||
private static string? NormalizeDigest(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = value.Trim();
|
||||
if (!trimmed.Contains(':', StringComparison.Ordinal))
|
||||
{
|
||||
trimmed = $"sha256:{trimmed}";
|
||||
}
|
||||
|
||||
return trimmed.ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -394,26 +394,59 @@ internal sealed class RuntimeInventoryReconciler : IRuntimeInventoryReconciler
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var options = _storageOptions.CurrentValue;
|
||||
var key = ArtifactObjectKeyBuilder.Build(
|
||||
|
||||
var primaryKey = ArtifactObjectKeyBuilder.Build(
|
||||
artifact.Type,
|
||||
artifact.Format,
|
||||
artifact.BytesSha256,
|
||||
options.ObjectStore.RootPrefix);
|
||||
|
||||
var descriptor = new ArtifactObjectDescriptor(
|
||||
options.ObjectStore.BucketName,
|
||||
key,
|
||||
artifact.Immutable);
|
||||
var candidates = new List<string>
|
||||
{
|
||||
primaryKey,
|
||||
ArtifactObjectKeyBuilder.Build(
|
||||
artifact.Type,
|
||||
artifact.Format,
|
||||
artifact.BytesSha256,
|
||||
rootPrefix: null)
|
||||
};
|
||||
|
||||
var legacyDigest = NormalizeLegacyDigest(artifact.BytesSha256);
|
||||
candidates.Add($"{MapLegacyTypeSegment(artifact.Type)}/{MapLegacyFormatSegment(artifact.Format)}/{legacyDigest}");
|
||||
|
||||
if (legacyDigest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
candidates.Add($"{MapLegacyTypeSegment(artifact.Type)}/{MapLegacyFormatSegment(artifact.Format)}/{legacyDigest["sha256:".Length..]}");
|
||||
}
|
||||
|
||||
Stream? stream = null;
|
||||
string? resolvedKey = null;
|
||||
|
||||
foreach (var candidateKey in candidates.Distinct(StringComparer.Ordinal))
|
||||
{
|
||||
var descriptor = new ArtifactObjectDescriptor(
|
||||
options.ObjectStore.BucketName,
|
||||
candidateKey,
|
||||
artifact.Immutable);
|
||||
|
||||
stream = await _objectStore.GetAsync(descriptor, cancellationToken).ConfigureAwait(false);
|
||||
if (stream is not null)
|
||||
{
|
||||
resolvedKey = candidateKey;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await using var stream = await _objectStore.GetAsync(descriptor, cancellationToken).ConfigureAwait(false);
|
||||
if (stream is null)
|
||||
{
|
||||
_logger.LogWarning("SBOM artifact content not found at {Key}", key);
|
||||
_logger.LogWarning("SBOM artifact content not found at {Key}", primaryKey);
|
||||
return [];
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await using (stream)
|
||||
{
|
||||
var bom = await Serializer.DeserializeAsync(stream).ConfigureAwait(false);
|
||||
if (bom?.Components is null)
|
||||
{
|
||||
@@ -435,10 +468,11 @@ internal sealed class RuntimeInventoryReconciler : IRuntimeInventoryReconciler
|
||||
FilePaths = ExtractFilePaths(c)
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to deserialize SBOM from artifact {ArtifactId}", artifact.Id);
|
||||
_logger.LogWarning(ex, "Failed to deserialize SBOM from artifact {ArtifactId} ({ResolvedKey})", artifact.Id, resolvedKey ?? primaryKey);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -595,6 +629,38 @@ internal sealed class RuntimeInventoryReconciler : IRuntimeInventoryReconciler
|
||||
return trimmed.ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string NormalizeLegacyDigest(string digest)
|
||||
=> digest.Contains(':', StringComparison.Ordinal)
|
||||
? digest.Trim()
|
||||
: $"sha256:{digest.Trim()}";
|
||||
|
||||
private static string MapLegacyTypeSegment(ArtifactDocumentType type) => type switch
|
||||
{
|
||||
ArtifactDocumentType.LayerBom => "layerbom",
|
||||
ArtifactDocumentType.ImageBom => "imagebom",
|
||||
ArtifactDocumentType.Index => "index",
|
||||
ArtifactDocumentType.Attestation => "attestation",
|
||||
ArtifactDocumentType.SurfaceManifest => "surface-manifest",
|
||||
ArtifactDocumentType.SurfaceEntryTrace => "surface-entrytrace",
|
||||
ArtifactDocumentType.SurfaceLayerFragment => "surface-layer-fragment",
|
||||
ArtifactDocumentType.Diff => "diff",
|
||||
_ => type.ToString().ToLowerInvariant()
|
||||
};
|
||||
|
||||
private static string MapLegacyFormatSegment(ArtifactDocumentFormat format) => format switch
|
||||
{
|
||||
ArtifactDocumentFormat.CycloneDxJson => "cyclonedx-json",
|
||||
ArtifactDocumentFormat.CycloneDxProtobuf => "cyclonedx-protobuf",
|
||||
ArtifactDocumentFormat.SpdxJson => "spdx-json",
|
||||
ArtifactDocumentFormat.BomIndex => "bom-index",
|
||||
ArtifactDocumentFormat.DsseJson => "dsse-json",
|
||||
ArtifactDocumentFormat.SurfaceManifestJson => "surface-manifest-json",
|
||||
ArtifactDocumentFormat.EntryTraceNdjson => "entrytrace-ndjson",
|
||||
ArtifactDocumentFormat.EntryTraceGraphJson => "entrytrace-graph-json",
|
||||
ArtifactDocumentFormat.ComponentFragmentJson => "component-fragment-json",
|
||||
_ => format.ToString().ToLowerInvariant()
|
||||
};
|
||||
|
||||
private static void RecordLatency(Stopwatch stopwatch)
|
||||
{
|
||||
stopwatch.Stop();
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
// Description: Service implementation for score replay operations
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Policy.Scoring;
|
||||
using StellaOps.Scanner.Core;
|
||||
|
||||
@@ -17,6 +17,7 @@ namespace StellaOps.Scanner.WebService.Services;
|
||||
/// </summary>
|
||||
public sealed class ScoreReplayService : IScoreReplayService
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _replayLocks = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly IScanManifestRepository _manifestRepository;
|
||||
private readonly IProofBundleRepository _bundleRepository;
|
||||
private readonly IProofBundleWriter _bundleWriter;
|
||||
@@ -49,52 +50,62 @@ public sealed class ScoreReplayService : IScoreReplayService
|
||||
{
|
||||
_logger.LogInformation("Starting score replay for scan {ScanId}", scanId);
|
||||
|
||||
// Get the manifest
|
||||
var signedManifest = await _manifestRepository.GetManifestAsync(scanId, manifestHash, cancellationToken);
|
||||
if (signedManifest is null)
|
||||
var replayLock = _replayLocks.GetOrAdd(scanId, _ => new SemaphoreSlim(1, 1));
|
||||
await replayLock.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogWarning("Manifest not found for scan {ScanId}", scanId);
|
||||
return null;
|
||||
}
|
||||
// Get the manifest
|
||||
var signedManifest = await _manifestRepository.GetManifestAsync(scanId, manifestHash, cancellationToken).ConfigureAwait(false);
|
||||
if (signedManifest is null)
|
||||
{
|
||||
_logger.LogWarning("Manifest not found for scan {ScanId}", scanId);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Verify manifest signature
|
||||
var verifyResult = await _manifestSigner.VerifyAsync(signedManifest, cancellationToken);
|
||||
if (!verifyResult.IsValid)
|
||||
// Verify manifest signature
|
||||
var verifyResult = await _manifestSigner.VerifyAsync(signedManifest, cancellationToken).ConfigureAwait(false);
|
||||
if (!verifyResult.IsValid)
|
||||
{
|
||||
throw new InvalidOperationException($"Manifest signature verification failed: {verifyResult.ErrorMessage}");
|
||||
}
|
||||
|
||||
var manifest = signedManifest.Manifest;
|
||||
|
||||
// Replay scoring with frozen inputs
|
||||
var ledger = new ProofLedger();
|
||||
var score = await _scoringService.ReplayScoreAsync(
|
||||
manifest.ScanId,
|
||||
manifest.ConcelierSnapshotHash,
|
||||
manifest.ExcititorSnapshotHash,
|
||||
manifest.LatticePolicyHash,
|
||||
manifest.Seed,
|
||||
freezeTimestamp ?? manifest.CreatedAtUtc,
|
||||
ledger,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Create proof bundle
|
||||
var bundle = await _bundleWriter.CreateBundleAsync(signedManifest, ledger, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Store bundle reference
|
||||
await _bundleRepository.SaveBundleAsync(bundle, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Score replay complete for scan {ScanId}: score={Score}, rootHash={RootHash}",
|
||||
scanId, score, bundle.RootHash);
|
||||
|
||||
return new ScoreReplayResult(
|
||||
Score: score,
|
||||
RootHash: bundle.RootHash,
|
||||
BundleUri: bundle.BundleUri,
|
||||
ManifestHash: manifest.ComputeHash(),
|
||||
ReplayedAt: DateTimeOffset.UtcNow,
|
||||
Deterministic: manifest.Deterministic);
|
||||
}
|
||||
finally
|
||||
{
|
||||
throw new InvalidOperationException($"Manifest signature verification failed: {verifyResult.ErrorMessage}");
|
||||
replayLock.Release();
|
||||
}
|
||||
|
||||
var manifest = signedManifest.Manifest;
|
||||
|
||||
// Replay scoring with frozen inputs
|
||||
var ledger = new ProofLedger();
|
||||
var score = await _scoringService.ReplayScoreAsync(
|
||||
manifest.ScanId,
|
||||
manifest.ConcelierSnapshotHash,
|
||||
manifest.ExcititorSnapshotHash,
|
||||
manifest.LatticePolicyHash,
|
||||
manifest.Seed,
|
||||
freezeTimestamp ?? manifest.CreatedAtUtc,
|
||||
ledger,
|
||||
cancellationToken);
|
||||
|
||||
// Create proof bundle
|
||||
var bundle = await _bundleWriter.CreateBundleAsync(signedManifest, ledger, cancellationToken);
|
||||
|
||||
// Store bundle reference
|
||||
await _bundleRepository.SaveBundleAsync(bundle, cancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Score replay complete for scan {ScanId}: score={Score}, rootHash={RootHash}",
|
||||
scanId, score, bundle.RootHash);
|
||||
|
||||
return new ScoreReplayResult(
|
||||
Score: score,
|
||||
RootHash: bundle.RootHash,
|
||||
BundleUri: bundle.BundleUri,
|
||||
ManifestHash: manifest.ComputeHash(),
|
||||
ReplayedAt: DateTimeOffset.UtcNow,
|
||||
Deterministic: manifest.Deterministic);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
| --- | --- | --- | --- |
|
||||
| `SCAN-API-3101-001` | `docs/implplan/archived/SPRINT_3101_0001_0001_scanner_api_standardization.md` | DOING | Align Scanner OpenAPI spec with current endpoints and include ProofSpine routes; compose into `src/Api/StellaOps.Api.OpenApi/stella.yaml`. |
|
||||
| `PROOFSPINE-3100-API` | `docs/implplan/archived/SPRINT_3100_0001_0001_proof_spine_system.md` | DONE | Implemented and tested `/api/v1/spines/*` endpoints with verification output (CBOR accept tracked in SPRINT_3105). |
|
||||
| `PROOF-CBOR-3105-001` | `docs/implplan/SPRINT_3105_0001_0001_proofspine_cbor_accept.md` | DOING | Add `Accept: application/cbor` support for ProofSpine endpoints + tests. |
|
||||
| `PROOF-CBOR-3105-001` | `docs/implplan/SPRINT_3105_0001_0001_proofspine_cbor_accept.md` | DONE | Added `Accept: application/cbor` support for ProofSpine endpoints + tests (`dotnet test src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj -c Release`). |
|
||||
| `SCAN-AIRGAP-0340-001` | `docs/implplan/SPRINT_0340_0001_0001_scanner_offline_config.md` | DONE | Offline kit import + DSSE/offline Rekor verification wired; integration tests cover success/failure/audit. |
|
||||
| `DRIFT-3600-API` | `docs/implplan/SPRINT_3600_0003_0001_drift_detection_engine.md` | DONE | Add reachability drift endpoints (`/api/v1/scans/{id}/drift`, `/api/v1/drift/{id}/sinks`) + integration tests. |
|
||||
| `SCAN-API-3103-001` | `docs/implplan/SPRINT_3103_0001_0001_scanner_api_ingestion_completion.md` | DONE | Implement missing ingestion services + DI for callgraph/SBOM endpoints and add deterministic integration tests. |
|
||||
| `EPSS-SCAN-011` | `docs/implplan/SPRINT_3410_0002_0001_epss_scanner_integration.md` | DONE | Wired `/api/v1/epss/*` endpoints and added `EpssEndpointsTests` integration coverage. |
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Diagnostics;
|
||||
|
||||
public static class EpssWorkerInstrumentation
|
||||
{
|
||||
public const string MeterName = "StellaOps.Scanner.Epss";
|
||||
|
||||
public static Meter Meter { get; } = new(MeterName, version: "1.0.0");
|
||||
}
|
||||
|
||||
@@ -6,10 +6,12 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Core.Epss;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Storage.Epss;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
|
||||
@@ -73,11 +75,6 @@ public sealed class EpssEnrichmentOptions
|
||||
EpssChangeFlags.CrossedHigh |
|
||||
EpssChangeFlags.BigJumpUp |
|
||||
EpssChangeFlags.BigJumpDown;
|
||||
|
||||
/// <summary>
|
||||
/// Suppress signals on model version change. Default: true.
|
||||
/// </summary>
|
||||
public bool SuppressSignalsOnModelChange { get; set; } = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -86,9 +83,27 @@ public sealed class EpssEnrichmentOptions
|
||||
/// </summary>
|
||||
public sealed class EpssEnrichmentJob : BackgroundService
|
||||
{
|
||||
private static readonly Counter<long> RunsTotal = EpssWorkerInstrumentation.Meter.CreateCounter<long>(
|
||||
"epss_enrichment_runs_total",
|
||||
description: "Number of EPSS enrichment job runs.");
|
||||
|
||||
private static readonly Histogram<double> DurationMs = EpssWorkerInstrumentation.Meter.CreateHistogram<double>(
|
||||
"epss_enrichment_duration_ms",
|
||||
unit: "ms",
|
||||
description: "EPSS enrichment job duration in milliseconds.");
|
||||
|
||||
private static readonly Counter<long> InstancesUpdatedTotal = EpssWorkerInstrumentation.Meter.CreateCounter<long>(
|
||||
"epss_enrichment_updated_total",
|
||||
description: "Number of vulnerability instances updated during EPSS enrichment (best-effort, depends on configured sink).");
|
||||
|
||||
private static readonly Counter<long> BandChangesTotal = EpssWorkerInstrumentation.Meter.CreateCounter<long>(
|
||||
"epss_enrichment_band_changes_total",
|
||||
description: "Number of EPSS priority band changes detected during enrichment.");
|
||||
|
||||
private readonly IEpssRepository _epssRepository;
|
||||
private readonly IEpssProvider _epssProvider;
|
||||
private readonly IEpssSignalPublisher _signalPublisher;
|
||||
private readonly EpssSignalJob? _signalJob;
|
||||
private readonly IOptions<EpssEnrichmentOptions> _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<EpssEnrichmentJob> _logger;
|
||||
@@ -103,11 +118,13 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
IEpssSignalPublisher signalPublisher,
|
||||
IOptions<EpssEnrichmentOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<EpssEnrichmentJob> logger)
|
||||
ILogger<EpssEnrichmentJob> logger,
|
||||
EpssSignalJob? signalJob = null)
|
||||
{
|
||||
_epssRepository = epssRepository ?? throw new ArgumentNullException(nameof(epssRepository));
|
||||
_epssProvider = epssProvider ?? throw new ArgumentNullException(nameof(epssProvider));
|
||||
_signalPublisher = signalPublisher ?? throw new ArgumentNullException(nameof(signalPublisher));
|
||||
_signalJob = signalJob;
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
@@ -167,6 +184,9 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
using var activity = _activitySource.StartActivity("epss.enrich", ActivityKind.Internal);
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var opts = _options.Value;
|
||||
DateOnly? modelDateForLog = null;
|
||||
var shouldTriggerSignals = false;
|
||||
var enrichmentSucceeded = false;
|
||||
|
||||
_logger.LogInformation("Starting EPSS enrichment");
|
||||
|
||||
@@ -177,9 +197,12 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
if (!modelDate.HasValue)
|
||||
{
|
||||
_logger.LogWarning("No EPSS data available for enrichment");
|
||||
RunsTotal.Add(1, new TagList { { "result", "skipped" } });
|
||||
return;
|
||||
}
|
||||
|
||||
modelDateForLog = modelDate.Value;
|
||||
shouldTriggerSignals = true;
|
||||
activity?.SetTag("epss.model_date", modelDate.Value.ToString("yyyy-MM-dd"));
|
||||
_logger.LogDebug("Using EPSS model date: {ModelDate}", modelDate.Value);
|
||||
|
||||
@@ -189,6 +212,8 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
if (changedCves.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No CVE changes to process");
|
||||
RunsTotal.Add(1, new TagList { { "result", "noop" } });
|
||||
enrichmentSucceeded = true;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -221,13 +246,28 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
activity?.SetTag("epss.updated_count", totalUpdated);
|
||||
activity?.SetTag("epss.band_change_count", totalBandChanges);
|
||||
activity?.SetTag("epss.duration_ms", stopwatch.ElapsedMilliseconds);
|
||||
|
||||
InstancesUpdatedTotal.Add(totalUpdated);
|
||||
BandChangesTotal.Add(totalBandChanges);
|
||||
DurationMs.Record(stopwatch.Elapsed.TotalMilliseconds);
|
||||
RunsTotal.Add(1, new TagList { { "result", "success" } });
|
||||
enrichmentSucceeded = true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "EPSS enrichment failed");
|
||||
activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
|
||||
RunsTotal.Add(1, new TagList { { "result", "failure" } });
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (shouldTriggerSignals && enrichmentSucceeded && _signalJob is not null)
|
||||
{
|
||||
_signalJob.TriggerSignalGeneration();
|
||||
_logger.LogDebug("Triggered EPSS signal generation for model date {ModelDate}", modelDateForLog);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<EpssChangeRecord>> GetChangedCvesAsync(
|
||||
@@ -238,7 +278,10 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
// Query epss_changes table for CVEs with matching flags for the model date (Task #4)
|
||||
_logger.LogDebug("Querying EPSS changes for model date {ModelDate} with flags {Flags}", modelDate, flags);
|
||||
|
||||
var changes = await _epssRepository.GetChangesAsync(modelDate, flags, cancellationToken: cancellationToken);
|
||||
var changes = await _epssRepository.GetChangesAsync(
|
||||
modelDate,
|
||||
flags: flags == EpssChangeFlags.None ? null : flags,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
_logger.LogDebug("Found {Count} EPSS changes matching flags {Flags}", changes.Count, flags);
|
||||
|
||||
@@ -311,7 +354,7 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
return EpssPriorityBand.Low;
|
||||
}
|
||||
|
||||
private Task EmitPriorityChangedEventAsync(
|
||||
private async Task EmitPriorityChangedEventAsync(
|
||||
string cveId,
|
||||
EpssPriorityBand previousBand,
|
||||
EpssPriorityBand newBand,
|
||||
@@ -335,7 +378,7 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
newBand.ToString(),
|
||||
evidence.Score,
|
||||
evidence.ModelDate,
|
||||
cancellationToken);
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.Success)
|
||||
{
|
||||
@@ -346,39 +389,3 @@ public sealed class EpssEnrichmentJob : BackgroundService
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Record representing an EPSS change that needs processing.
|
||||
/// </summary>
|
||||
public sealed record EpssChangeRecord
|
||||
{
|
||||
/// <summary>
|
||||
/// CVE identifier.
|
||||
/// </summary>
|
||||
public required string CveId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Change flags indicating what changed.
|
||||
/// </summary>
|
||||
public EpssChangeFlags Flags { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Previous EPSS score (if available).
|
||||
/// </summary>
|
||||
public double? PreviousScore { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// New EPSS score.
|
||||
/// </summary>
|
||||
public double NewScore { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Previous priority band (if available).
|
||||
/// </summary>
|
||||
public EpssPriorityBand PreviousBand { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Model date for this change.
|
||||
/// </summary>
|
||||
public DateOnly ModelDate { get; init; }
|
||||
}
|
||||
|
||||
@@ -88,29 +88,28 @@ public sealed class EpssEnrichmentStageExecutor : IScanStageExecutor
|
||||
var cveIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Extract from OS package analyzer results
|
||||
if (context.Analysis.TryGet<Dictionary<string, object>>(ScanAnalysisKeys.OsPackageAnalyzers, out var osResults) && osResults is not null)
|
||||
if (context.Analysis.TryGet<object>(ScanAnalysisKeys.OsPackageAnalyzers, out var osResults) &&
|
||||
osResults is System.Collections.IDictionary osDictionary)
|
||||
{
|
||||
foreach (var analyzerResult in osResults.Values)
|
||||
foreach (var analyzerResult in osDictionary.Values)
|
||||
{
|
||||
ExtractCvesFromAnalyzerResult(analyzerResult, cveIds);
|
||||
if (analyzerResult is not null)
|
||||
{
|
||||
ExtractCvesFromAnalyzerResult(analyzerResult, cveIds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract from language analyzer results
|
||||
if (context.Analysis.TryGet<Dictionary<string, object>>(ScanAnalysisKeys.LanguagePackageAnalyzers, out var langResults) && langResults is not null)
|
||||
if (context.Analysis.TryGet<object>(ScanAnalysisKeys.LanguageAnalyzerResults, out var langResults) &&
|
||||
langResults is System.Collections.IDictionary langDictionary)
|
||||
{
|
||||
foreach (var analyzerResult in langResults.Values)
|
||||
foreach (var analyzerResult in langDictionary.Values)
|
||||
{
|
||||
ExtractCvesFromAnalyzerResult(analyzerResult, cveIds);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract from consolidated findings if available
|
||||
if (context.Analysis.TryGet<IEnumerable<object>>(ScanAnalysisKeys.ConsolidatedFindings, out var findings) && findings is not null)
|
||||
{
|
||||
foreach (var finding in findings)
|
||||
{
|
||||
ExtractCvesFromFinding(finding, cveIds);
|
||||
if (analyzerResult is not null)
|
||||
{
|
||||
ExtractCvesFromAnalyzerResult(analyzerResult, cveIds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,24 +181,3 @@ public sealed class EpssEnrichmentStageExecutor : IScanStageExecutor
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Well-known keys for EPSS-related analysis data.
|
||||
/// </summary>
|
||||
public static partial class ScanAnalysisKeys
|
||||
{
|
||||
/// <summary>
|
||||
/// Dictionary of CVE ID to EpssEvidence for enriched findings.
|
||||
/// </summary>
|
||||
public const string EpssEvidence = "epss.evidence";
|
||||
|
||||
/// <summary>
|
||||
/// The EPSS model date used for enrichment.
|
||||
/// </summary>
|
||||
public const string EpssModelDate = "epss.model_date";
|
||||
|
||||
/// <summary>
|
||||
/// List of CVE IDs that were not found in EPSS data.
|
||||
/// </summary>
|
||||
public const string EpssNotFoundCves = "epss.not_found";
|
||||
}
|
||||
|
||||
@@ -6,6 +6,9 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -68,6 +71,7 @@ public sealed class EpssIngestJob : BackgroundService
|
||||
{
|
||||
private readonly IEpssRepository _repository;
|
||||
private readonly IEpssRawRepository? _rawRepository;
|
||||
private readonly EpssEnrichmentJob? _enrichmentJob;
|
||||
private readonly EpssOnlineSource _onlineSource;
|
||||
private readonly EpssBundleSource _bundleSource;
|
||||
private readonly EpssCsvStreamParser _parser;
|
||||
@@ -84,10 +88,12 @@ public sealed class EpssIngestJob : BackgroundService
|
||||
IOptions<EpssIngestOptions> options,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<EpssIngestJob> logger,
|
||||
IEpssRawRepository? rawRepository = null)
|
||||
IEpssRawRepository? rawRepository = null,
|
||||
EpssEnrichmentJob? enrichmentJob = null)
|
||||
{
|
||||
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
|
||||
_rawRepository = rawRepository; // Optional - raw storage for replay capability
|
||||
_enrichmentJob = enrichmentJob; // Optional - live enrichment trigger
|
||||
_onlineSource = onlineSource ?? throw new ArgumentNullException(nameof(onlineSource));
|
||||
_bundleSource = bundleSource ?? throw new ArgumentNullException(nameof(bundleSource));
|
||||
_parser = parser ?? throw new ArgumentNullException(nameof(parser));
|
||||
@@ -174,29 +180,43 @@ public sealed class EpssIngestJob : BackgroundService
|
||||
fileSha256,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogDebug("Created import run {ImportRunId}", importRun.ImportRunId);
|
||||
_logger.LogDebug("Created import run {ImportRunId}", importRun.ImportRunId);
|
||||
|
||||
try
|
||||
{
|
||||
// Parse and write snapshot
|
||||
await using var stream = new MemoryStream(fileContent);
|
||||
var session = _parser.ParseGzip(stream);
|
||||
await using var session = _parser.ParseGzip(stream);
|
||||
|
||||
System.Buffers.ArrayBufferWriter<byte>? rawPayloadBuffer = null;
|
||||
Utf8JsonWriter? rawPayloadWriter = null;
|
||||
|
||||
var rows = (IAsyncEnumerable<EpssScoreRow>)session;
|
||||
if (_rawRepository is not null)
|
||||
{
|
||||
rawPayloadBuffer = new System.Buffers.ArrayBufferWriter<byte>();
|
||||
rawPayloadWriter = new Utf8JsonWriter(rawPayloadBuffer, new JsonWriterOptions { Indented = false });
|
||||
rows = TeeRowsWithRawCaptureAsync(session, rawPayloadWriter, cancellationToken);
|
||||
}
|
||||
|
||||
var writeResult = await _repository.WriteSnapshotAsync(
|
||||
importRun.ImportRunId,
|
||||
modelDate,
|
||||
_timeProvider.GetUtcNow(),
|
||||
session,
|
||||
rows,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Store raw payload for replay capability (Sprint: SPRINT_3413_0001_0001, Task: R2)
|
||||
if (_rawRepository is not null)
|
||||
if (_rawRepository is not null && rawPayloadBuffer is not null)
|
||||
{
|
||||
rawPayloadWriter?.Dispose();
|
||||
|
||||
await StoreRawPayloadAsync(
|
||||
importRun.ImportRunId,
|
||||
sourceFile.SourceUri,
|
||||
modelDate,
|
||||
session,
|
||||
rawPayloadBuffer.WrittenMemory,
|
||||
fileContent.Length,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
@@ -222,6 +242,15 @@ public sealed class EpssIngestJob : BackgroundService
|
||||
activity?.SetTag("epss.row_count", writeResult.RowCount);
|
||||
activity?.SetTag("epss.cve_count", writeResult.DistinctCveCount);
|
||||
activity?.SetTag("epss.duration_ms", stopwatch.ElapsedMilliseconds);
|
||||
|
||||
if (_enrichmentJob is not null)
|
||||
{
|
||||
_enrichmentJob.TriggerEnrichment();
|
||||
_logger.LogDebug(
|
||||
"Triggered EPSS enrichment for {ModelDate} after import run {ImportRunId}",
|
||||
modelDate,
|
||||
importRun.ImportRunId);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -303,7 +332,8 @@ public sealed class EpssIngestJob : BackgroundService
|
||||
Guid importRunId,
|
||||
string sourceUri,
|
||||
DateOnly modelDate,
|
||||
EpssParsedSession session,
|
||||
EpssCsvStreamParser.EpssCsvParseSession session,
|
||||
ReadOnlyMemory<byte> payloadBytes,
|
||||
long compressedSize,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -314,18 +344,8 @@ public sealed class EpssIngestJob : BackgroundService
|
||||
|
||||
try
|
||||
{
|
||||
// Convert parsed rows to JSON array for raw storage
|
||||
var payload = System.Text.Json.JsonSerializer.Serialize(
|
||||
session.Rows.Select(r => new
|
||||
{
|
||||
cve = r.CveId,
|
||||
epss = r.Score,
|
||||
percentile = r.Percentile
|
||||
}),
|
||||
new System.Text.Json.JsonSerializerOptions { WriteIndented = false });
|
||||
|
||||
var payloadBytes = System.Text.Encoding.UTF8.GetBytes(payload);
|
||||
var payloadSha256 = System.Security.Cryptography.SHA256.HashData(payloadBytes);
|
||||
var payloadSha256 = System.Security.Cryptography.SHA256.HashData(payloadBytes.Span);
|
||||
var payload = Encoding.UTF8.GetString(payloadBytes.Span);
|
||||
|
||||
var raw = new EpssRaw
|
||||
{
|
||||
@@ -333,12 +353,11 @@ public sealed class EpssIngestJob : BackgroundService
|
||||
AsOfDate = modelDate,
|
||||
Payload = payload,
|
||||
PayloadSha256 = payloadSha256,
|
||||
HeaderComment = session.HeaderComment,
|
||||
ModelVersion = session.ModelVersionTag,
|
||||
PublishedDate = session.PublishedDate,
|
||||
RowCount = session.RowCount,
|
||||
CompressedSize = compressedSize,
|
||||
DecompressedSize = payloadBytes.LongLength,
|
||||
DecompressedSize = payloadBytes.Length,
|
||||
ImportRunId = importRunId
|
||||
};
|
||||
|
||||
@@ -359,4 +378,28 @@ public sealed class EpssIngestJob : BackgroundService
|
||||
modelDate);
|
||||
}
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<EpssScoreRow> TeeRowsWithRawCaptureAsync(
|
||||
IAsyncEnumerable<EpssScoreRow> rows,
|
||||
Utf8JsonWriter writer,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
writer.WriteStartArray();
|
||||
|
||||
await foreach (var row in rows.WithCancellation(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
writer.WriteStartObject();
|
||||
writer.WriteString("cve", row.CveId);
|
||||
writer.WriteNumber("epss", row.Score);
|
||||
writer.WriteNumber("percentile", row.Percentile);
|
||||
writer.WriteEndObject();
|
||||
|
||||
yield return row;
|
||||
}
|
||||
|
||||
writer.WriteEndArray();
|
||||
await writer.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -13,6 +14,7 @@ using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Core.Epss;
|
||||
using StellaOps.Scanner.Storage.Epss;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
@@ -45,6 +47,11 @@ public sealed class EpssSignalOptions
|
||||
/// Signal retention days. Default: 90.
|
||||
/// </summary>
|
||||
public int RetentionDays { get; set; } = 90;
|
||||
|
||||
/// <summary>
|
||||
/// Suppress individual signals on model version change days. Default: true.
|
||||
/// </summary>
|
||||
public bool SuppressSignalsOnModelChange { get; set; } = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -84,6 +91,19 @@ public static class EpssSignalEventTypes
|
||||
/// </summary>
|
||||
public sealed class EpssSignalJob : BackgroundService
|
||||
{
|
||||
private static readonly Counter<long> RunsTotal = EpssWorkerInstrumentation.Meter.CreateCounter<long>(
|
||||
"epss_signal_runs_total",
|
||||
description: "Number of EPSS signal generation job runs.");
|
||||
|
||||
private static readonly Histogram<double> DurationMs = EpssWorkerInstrumentation.Meter.CreateHistogram<double>(
|
||||
"epss_signal_duration_ms",
|
||||
unit: "ms",
|
||||
description: "EPSS signal generation job duration in milliseconds.");
|
||||
|
||||
private static readonly Counter<long> SignalsEmittedTotal = EpssWorkerInstrumentation.Meter.CreateCounter<long>(
|
||||
"epss_signals_emitted_total",
|
||||
description: "Number of EPSS signals emitted, labeled by event type and tenant.");
|
||||
|
||||
private readonly IEpssRepository _epssRepository;
|
||||
private readonly IEpssSignalRepository _signalRepository;
|
||||
private readonly IObservedCveRepository _observedCveRepository;
|
||||
@@ -177,6 +197,7 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
using var activity = _activitySource.StartActivity("epss.signal.generate", ActivityKind.Internal);
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var opts = _options.Value;
|
||||
var suppressSignalsOnModelChange = opts.SuppressSignalsOnModelChange;
|
||||
|
||||
_logger.LogInformation("Starting EPSS signal generation");
|
||||
|
||||
@@ -187,21 +208,24 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
if (!modelDate.HasValue)
|
||||
{
|
||||
_logger.LogWarning("No EPSS data available for signal generation");
|
||||
RunsTotal.Add(1, new TagList { { "result", "skipped" } });
|
||||
return;
|
||||
}
|
||||
|
||||
activity?.SetTag("epss.model_date", modelDate.Value.ToString("yyyy-MM-dd"));
|
||||
|
||||
// Check for model version change (S7)
|
||||
var previousModelVersion = _lastModelVersion;
|
||||
var currentModelVersion = await GetCurrentModelVersionAsync(modelDate.Value, cancellationToken);
|
||||
var isModelChange = _lastModelVersion is not null &&
|
||||
!string.Equals(_lastModelVersion, currentModelVersion, StringComparison.Ordinal);
|
||||
var isModelChange = previousModelVersion is not null &&
|
||||
currentModelVersion is not null &&
|
||||
!string.Equals(previousModelVersion, currentModelVersion, StringComparison.Ordinal);
|
||||
|
||||
if (isModelChange)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"EPSS model version changed: {OldVersion} -> {NewVersion}",
|
||||
_lastModelVersion,
|
||||
previousModelVersion,
|
||||
currentModelVersion);
|
||||
}
|
||||
|
||||
@@ -212,6 +236,7 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
if (changes.Count == 0)
|
||||
{
|
||||
_logger.LogDebug("No EPSS changes to process for signals");
|
||||
RunsTotal.Add(1, new TagList { { "result", "noop" } });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -251,7 +276,7 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
continue;
|
||||
}
|
||||
|
||||
filteredCount += changes.Length - tenantChanges.Length;
|
||||
filteredCount += changes.Count - tenantChanges.Length;
|
||||
|
||||
foreach (var batch in tenantChanges.Chunk(opts.BatchSize))
|
||||
{
|
||||
@@ -275,20 +300,36 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
published,
|
||||
signals.Count,
|
||||
tenantId);
|
||||
|
||||
foreach (var signal in signals)
|
||||
{
|
||||
SignalsEmittedTotal.Add(1, new TagList
|
||||
{
|
||||
{ "event_type", signal.EventType },
|
||||
{ "tenant_id", signal.TenantId.ToString("D") }
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If model changed, emit summary signal per tenant (S8)
|
||||
if (isModelChange)
|
||||
if (isModelChange && previousModelVersion is not null && currentModelVersion is not null)
|
||||
{
|
||||
await EmitModelUpdatedSignalAsync(
|
||||
tenantId,
|
||||
modelDate.Value,
|
||||
_lastModelVersion!,
|
||||
currentModelVersion!,
|
||||
previousModelVersion,
|
||||
currentModelVersion,
|
||||
suppressedSignals: suppressSignalsOnModelChange,
|
||||
tenantChanges.Length,
|
||||
cancellationToken);
|
||||
totalSignals++;
|
||||
|
||||
SignalsEmittedTotal.Add(1, new TagList
|
||||
{
|
||||
{ "event_type", EpssSignalEventTypes.ModelUpdated },
|
||||
{ "tenant_id", tenantId.ToString("D") }
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -306,11 +347,15 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
activity?.SetTag("epss.filtered_count", filteredCount);
|
||||
activity?.SetTag("epss.tenant_count", activeTenants.Count);
|
||||
activity?.SetTag("epss.duration_ms", stopwatch.ElapsedMilliseconds);
|
||||
|
||||
DurationMs.Record(stopwatch.Elapsed.TotalMilliseconds);
|
||||
RunsTotal.Add(1, new TagList { { "result", "success" } });
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "EPSS signal generation failed");
|
||||
activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
|
||||
RunsTotal.Add(1, new TagList { { "result", "failure" } });
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -322,18 +367,20 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
string? modelVersion,
|
||||
bool isModelChange)
|
||||
{
|
||||
var suppressSignalsOnModelChange = _options.Value.SuppressSignalsOnModelChange;
|
||||
var signals = new List<EpssSignal>();
|
||||
|
||||
foreach (var change in changes)
|
||||
{
|
||||
// Skip generating individual signals on model change day if suppression is enabled
|
||||
// (would check tenant config in production)
|
||||
if (isModelChange && ShouldSuppressOnModelChange(change))
|
||||
if (isModelChange && suppressSignalsOnModelChange && ShouldSuppressOnModelChange(change))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var eventType = DetermineEventType(change);
|
||||
var newBand = ComputeNewBand(change.NewPercentile);
|
||||
var eventType = DetermineEventType(change, newBand);
|
||||
if (string.IsNullOrEmpty(eventType))
|
||||
{
|
||||
continue;
|
||||
@@ -344,16 +391,16 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
change.CveId,
|
||||
eventType,
|
||||
change.PreviousBand.ToString(),
|
||||
ComputeNewBand(change).ToString());
|
||||
newBand.ToString());
|
||||
|
||||
var explainHash = EpssExplainHashCalculator.ComputeExplainHash(
|
||||
modelDate,
|
||||
change.CveId,
|
||||
eventType,
|
||||
change.PreviousBand.ToString(),
|
||||
ComputeNewBand(change).ToString(),
|
||||
newBand.ToString(),
|
||||
change.NewScore,
|
||||
0, // Percentile would come from EPSS data
|
||||
change.NewPercentile,
|
||||
modelVersion);
|
||||
|
||||
var payload = JsonSerializer.Serialize(new
|
||||
@@ -362,20 +409,23 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
oldScore = change.PreviousScore,
|
||||
newScore = change.NewScore,
|
||||
oldBand = change.PreviousBand.ToString(),
|
||||
newBand = ComputeNewBand(change).ToString(),
|
||||
newBand = newBand.ToString(),
|
||||
flags = change.Flags.ToString(),
|
||||
modelVersion
|
||||
});
|
||||
|
||||
double? delta = change.PreviousScore is null ? null : change.NewScore - change.PreviousScore.Value;
|
||||
|
||||
signals.Add(new EpssSignal
|
||||
{
|
||||
TenantId = tenantId,
|
||||
ModelDate = modelDate,
|
||||
CveId = change.CveId,
|
||||
EventType = eventType,
|
||||
RiskBand = ComputeNewBand(change).ToString(),
|
||||
RiskBand = newBand.ToString(),
|
||||
EpssScore = change.NewScore,
|
||||
EpssDelta = change.NewScore - (change.PreviousScore ?? 0),
|
||||
EpssDelta = delta,
|
||||
Percentile = change.NewPercentile,
|
||||
IsModelChange = isModelChange,
|
||||
ModelVersion = modelVersion,
|
||||
DedupeKey = dedupeKey,
|
||||
@@ -387,45 +437,44 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
return signals;
|
||||
}
|
||||
|
||||
private static string? DetermineEventType(EpssChangeRecord change)
|
||||
private static string? DetermineEventType(EpssChangeRecord change, EpssPriorityBand newBand)
|
||||
{
|
||||
if (change.Flags.HasFlag(EpssChangeFlags.NewScored))
|
||||
{
|
||||
return EpssSignalEventTypes.NewHigh;
|
||||
}
|
||||
|
||||
if (change.Flags.HasFlag(EpssChangeFlags.CrossedHigh))
|
||||
{
|
||||
return EpssSignalEventTypes.BandChange;
|
||||
}
|
||||
|
||||
if (change.Flags.HasFlag(EpssChangeFlags.BigJumpUp))
|
||||
{
|
||||
return EpssSignalEventTypes.RiskSpike;
|
||||
}
|
||||
|
||||
if (change.Flags.HasFlag(EpssChangeFlags.BigJumpDown))
|
||||
if (change.Flags.HasFlag(EpssChangeFlags.BigJumpDown) || change.Flags.HasFlag(EpssChangeFlags.CrossedLow))
|
||||
{
|
||||
return EpssSignalEventTypes.DroppedLow;
|
||||
}
|
||||
|
||||
if (change.PreviousBand != newBand || change.Flags.HasFlag(EpssChangeFlags.CrossedHigh))
|
||||
{
|
||||
return EpssSignalEventTypes.BandChange;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static EpssPriorityBand ComputeNewBand(EpssChangeRecord change)
|
||||
private static EpssPriorityBand ComputeNewBand(double percentile)
|
||||
{
|
||||
// Simplified band calculation - would use EpssPriorityCalculator in production
|
||||
if (change.NewScore >= 0.5)
|
||||
if (percentile >= 0.995)
|
||||
{
|
||||
return EpssPriorityBand.Critical;
|
||||
}
|
||||
|
||||
if (change.NewScore >= 0.2)
|
||||
if (percentile >= 0.99)
|
||||
{
|
||||
return EpssPriorityBand.High;
|
||||
}
|
||||
|
||||
if (change.NewScore >= 0.05)
|
||||
if (percentile >= 0.90)
|
||||
{
|
||||
return EpssPriorityBand.Medium;
|
||||
}
|
||||
@@ -443,18 +492,17 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
|
||||
private async Task<string?> GetCurrentModelVersionAsync(DateOnly modelDate, CancellationToken cancellationToken)
|
||||
{
|
||||
// Would query from epss_import_run or epss_raw table
|
||||
// For now, return a placeholder based on date
|
||||
return $"v{modelDate:yyyy.MM.dd}";
|
||||
var run = await _epssRepository.GetImportRunAsync(modelDate, cancellationToken).ConfigureAwait(false);
|
||||
return string.IsNullOrWhiteSpace(run?.ModelVersionTag)
|
||||
? $"v{modelDate:yyyy.MM.dd}"
|
||||
: run.ModelVersionTag;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<EpssChangeRecord>> GetEpssChangesAsync(
|
||||
DateOnly modelDate,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// TODO: Implement repository method to get changes from epss_changes table
|
||||
// For now, return empty list
|
||||
return Array.Empty<EpssChangeRecord>();
|
||||
return await _epssRepository.GetChangesAsync(modelDate, flags: null, limit: 200000, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task EmitModelUpdatedSignalAsync(
|
||||
@@ -462,6 +510,7 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
DateOnly modelDate,
|
||||
string oldVersion,
|
||||
string newVersion,
|
||||
bool suppressedSignals,
|
||||
int affectedCveCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -470,7 +519,7 @@ public sealed class EpssSignalJob : BackgroundService
|
||||
oldVersion,
|
||||
newVersion,
|
||||
affectedCveCount,
|
||||
suppressedSignals = true
|
||||
suppressedSignals
|
||||
});
|
||||
|
||||
var signal = new EpssSignal
|
||||
|
||||
@@ -119,6 +119,19 @@ if (!string.IsNullOrWhiteSpace(connectionString))
|
||||
.BindConfiguration(EpssIngestOptions.SectionName)
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddHostedService<EpssIngestJob>();
|
||||
|
||||
// EPSS live enrichment + signals (Sprint: SPRINT_3413_0001_0001)
|
||||
builder.Services.AddOptions<EpssEnrichmentOptions>()
|
||||
.BindConfiguration(EpssEnrichmentOptions.SectionName)
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddSingleton<EpssEnrichmentJob>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<EpssEnrichmentJob>());
|
||||
|
||||
builder.Services.AddOptions<EpssSignalOptions>()
|
||||
.BindConfiguration(EpssSignalOptions.SectionName)
|
||||
.ValidateOnStart();
|
||||
builder.Services.AddSingleton<EpssSignalJob>();
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<EpssSignalJob>());
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
using System.Globalization;
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Epss.Perf;
|
||||
|
||||
internal sealed record GeneratedEpssDataset(byte[] GzipBytes, long DecompressedBytes);
|
||||
|
||||
internal static class EpssDatasetGenerator
|
||||
{
|
||||
public static GeneratedEpssDataset GenerateGzip(DateOnly modelDate, int rowCount, ulong seed)
|
||||
{
|
||||
if (rowCount < 1)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(rowCount), rowCount, "Row count must be positive.");
|
||||
}
|
||||
|
||||
using var raw = new MemoryStream(capacity: Math.Min(64 * 1024 * 1024, rowCount * 48));
|
||||
using (var gzip = new GZipStream(raw, CompressionLevel.SmallestSize, leaveOpen: true))
|
||||
using (var writer = new StreamWriter(gzip, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), bufferSize: 64 * 1024, leaveOpen: true))
|
||||
{
|
||||
writer.NewLine = "\n";
|
||||
|
||||
var versionTag = $"v{modelDate:yyyy.MM.dd}";
|
||||
writer.Write("# EPSS model ");
|
||||
writer.Write(versionTag);
|
||||
writer.Write(" published ");
|
||||
writer.WriteLine(modelDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
|
||||
|
||||
writer.WriteLine("cve,epss,percentile");
|
||||
|
||||
var prng = new XorShift64Star(seed);
|
||||
long decompressedBytes = 0;
|
||||
|
||||
for (var i = 0; i < rowCount; i++)
|
||||
{
|
||||
var cve = $"CVE-2024-{(i + 1):D7}";
|
||||
var score = prng.NextDouble();
|
||||
var percentile = prng.NextDouble();
|
||||
|
||||
// Keep formatting deterministic and compact.
|
||||
var line = string.Create(CultureInfo.InvariantCulture, $"{cve},{score:0.000000},{percentile:0.000000}\n");
|
||||
decompressedBytes += Encoding.UTF8.GetByteCount(line);
|
||||
writer.Write(line);
|
||||
}
|
||||
|
||||
writer.Flush();
|
||||
gzip.Flush();
|
||||
|
||||
return new GeneratedEpssDataset(raw.ToArray(), decompressedBytes);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class XorShift64Star
|
||||
{
|
||||
private ulong _state;
|
||||
|
||||
public XorShift64Star(ulong seed)
|
||||
{
|
||||
_state = seed == 0 ? 0x9E3779B97F4A7C15UL : seed;
|
||||
}
|
||||
|
||||
private ulong NextUInt64()
|
||||
{
|
||||
// xorshift64*
|
||||
var x = _state;
|
||||
x ^= x >> 12;
|
||||
x ^= x << 25;
|
||||
x ^= x >> 27;
|
||||
_state = x;
|
||||
return x * 0x2545F4914F6CDD1DUL;
|
||||
}
|
||||
|
||||
public double NextDouble()
|
||||
{
|
||||
// Build a double in [0,1) with 53 bits of precision.
|
||||
var value = NextUInt64() >> 11;
|
||||
return value * (1.0 / (1UL << 53));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
using System.Diagnostics;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Infrastructure.Postgres.Testing;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.Epss.Perf;
|
||||
using StellaOps.Scanner.Storage.Epss;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
|
||||
using Testcontainers.PostgreSql;
|
||||
|
||||
var options = PerfOptions.Parse(args);
|
||||
var outputDirectory = Path.GetDirectoryName(options.OutputPath);
|
||||
if (!string.IsNullOrWhiteSpace(outputDirectory))
|
||||
{
|
||||
Directory.CreateDirectory(outputDirectory);
|
||||
}
|
||||
|
||||
var result = await RunAsync(options, CancellationToken.None).ConfigureAwait(false);
|
||||
var json = JsonSerializer.Serialize(
|
||||
result,
|
||||
new JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
});
|
||||
await File.WriteAllTextAsync(options.OutputPath, json, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)).ConfigureAwait(false);
|
||||
|
||||
static async Task<EpssIngestPerfResult> RunAsync(PerfOptions options, CancellationToken cancellationToken)
|
||||
{
|
||||
var overallStopwatch = Stopwatch.StartNew();
|
||||
|
||||
var datasetStopwatch = Stopwatch.StartNew();
|
||||
var dataset = EpssDatasetGenerator.GenerateGzip(
|
||||
options.ModelDate,
|
||||
options.RowCount,
|
||||
options.Seed);
|
||||
datasetStopwatch.Stop();
|
||||
|
||||
var compressedSha256 = "sha256:" + Convert.ToHexString(SHA256.HashData(dataset.GzipBytes)).ToLowerInvariant();
|
||||
|
||||
var containerStopwatch = Stopwatch.StartNew();
|
||||
await using var container = new PostgreSqlBuilder()
|
||||
.WithImage(options.PostgresImage)
|
||||
.Build();
|
||||
await container.StartAsync(cancellationToken).ConfigureAwait(false);
|
||||
containerStopwatch.Stop();
|
||||
|
||||
var fixture = PostgresFixtureFactory.CreateRandom(container.GetConnectionString(), NullLogger.Instance);
|
||||
await fixture.InitializeAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var migrationsStopwatch = Stopwatch.StartNew();
|
||||
await fixture.RunMigrationsFromAssemblyAsync(
|
||||
typeof(ScannerStorageOptions).Assembly,
|
||||
moduleName: "Scanner.Storage",
|
||||
resourcePrefix: null,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
migrationsStopwatch.Stop();
|
||||
|
||||
var storageOptions = new ScannerStorageOptions
|
||||
{
|
||||
Postgres = new StellaOps.Infrastructure.Postgres.Options.PostgresOptions
|
||||
{
|
||||
ConnectionString = container.GetConnectionString(),
|
||||
SchemaName = fixture.SchemaName
|
||||
}
|
||||
};
|
||||
|
||||
var dataSource = new ScannerDataSource(
|
||||
Options.Create(storageOptions),
|
||||
NullLogger<ScannerDataSource>.Instance);
|
||||
|
||||
var repository = new PostgresEpssRepository(dataSource);
|
||||
var parser = new EpssCsvStreamParser();
|
||||
|
||||
var retrievedAt = DateTimeOffset.UtcNow;
|
||||
var importRun = await repository.BeginImportAsync(
|
||||
options.ModelDate,
|
||||
sourceUri: $"perf://generated?rows={options.RowCount}",
|
||||
retrievedAtUtc: retrievedAt,
|
||||
fileSha256: compressedSha256,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var writeStopwatch = Stopwatch.StartNew();
|
||||
await using var parseSession = parser.ParseGzip(new MemoryStream(dataset.GzipBytes, writable: false));
|
||||
var writeResult = await repository.WriteSnapshotAsync(
|
||||
importRun.ImportRunId,
|
||||
options.ModelDate,
|
||||
updatedAtUtc: retrievedAt,
|
||||
rows: parseSession,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
writeStopwatch.Stop();
|
||||
|
||||
await repository.MarkImportSucceededAsync(
|
||||
importRun.ImportRunId,
|
||||
rowCount: writeResult.RowCount,
|
||||
decompressedSha256: parseSession.DecompressedSha256,
|
||||
modelVersionTag: parseSession.ModelVersionTag,
|
||||
publishedDate: parseSession.PublishedDate,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
overallStopwatch.Stop();
|
||||
|
||||
await fixture.DisposeAsync().ConfigureAwait(false);
|
||||
|
||||
return new EpssIngestPerfResult
|
||||
{
|
||||
Tool = new PerfToolInfo
|
||||
{
|
||||
Name = "StellaOps.Scanner.Storage.Epss.Perf",
|
||||
Schema = 1
|
||||
},
|
||||
Dataset = new PerfDatasetInfo
|
||||
{
|
||||
ModelDate = options.ModelDate.ToString("yyyy-MM-dd"),
|
||||
Rows = options.RowCount,
|
||||
Seed = options.Seed,
|
||||
CompressedSha256 = compressedSha256,
|
||||
DecompressedSha256 = parseSession.DecompressedSha256,
|
||||
ModelVersionTag = parseSession.ModelVersionTag,
|
||||
PublishedDate = parseSession.PublishedDate?.ToString("yyyy-MM-dd"),
|
||||
CompressedBytes = dataset.GzipBytes.LongLength,
|
||||
DecompressedBytes = dataset.DecompressedBytes
|
||||
},
|
||||
Environment = new PerfEnvironmentInfo
|
||||
{
|
||||
Os = Environment.OSVersion.ToString(),
|
||||
Framework = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription,
|
||||
ProcessArchitecture = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture.ToString(),
|
||||
PostgresImage = options.PostgresImage
|
||||
},
|
||||
TimingsMs = new PerfTimingInfo
|
||||
{
|
||||
DatasetGenerate = datasetStopwatch.ElapsedMilliseconds,
|
||||
ContainerStart = containerStopwatch.ElapsedMilliseconds,
|
||||
Migrations = migrationsStopwatch.ElapsedMilliseconds,
|
||||
WriteSnapshot = writeStopwatch.ElapsedMilliseconds,
|
||||
Total = overallStopwatch.ElapsedMilliseconds
|
||||
},
|
||||
Result = new PerfWriteResultInfo
|
||||
{
|
||||
ImportRunId = importRun.ImportRunId,
|
||||
RowCount = writeResult.RowCount,
|
||||
DistinctCveCount = writeResult.DistinctCveCount
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
internal sealed record PerfOptions(DateOnly ModelDate, int RowCount, ulong Seed, string PostgresImage, string OutputPath)
|
||||
{
|
||||
public static PerfOptions Parse(string[] args)
|
||||
{
|
||||
var modelDate = DateOnly.FromDateTime(DateTime.UtcNow.Date);
|
||||
var rowCount = 310_000;
|
||||
ulong seed = 0x5EED_2025_12_19;
|
||||
var postgresImage = "postgres:16-alpine";
|
||||
var outputPath = Path.Combine("bench", "results", "epss-ingest-perf.json");
|
||||
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
var arg = args[i];
|
||||
if (string.Equals(arg, "--rows", StringComparison.OrdinalIgnoreCase) && i + 1 < args.Length)
|
||||
{
|
||||
rowCount = int.Parse(args[++i]);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(arg, "--seed", StringComparison.OrdinalIgnoreCase) && i + 1 < args.Length)
|
||||
{
|
||||
seed = Convert.ToUInt64(args[++i], 16);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(arg, "--model-date", StringComparison.OrdinalIgnoreCase) && i + 1 < args.Length)
|
||||
{
|
||||
modelDate = DateOnly.Parse(args[++i]);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(arg, "--postgres-image", StringComparison.OrdinalIgnoreCase) && i + 1 < args.Length)
|
||||
{
|
||||
postgresImage = args[++i];
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(arg, "--output", StringComparison.OrdinalIgnoreCase) && i + 1 < args.Length)
|
||||
{
|
||||
outputPath = args[++i];
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(arg, "--help", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(arg, "-h", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
Console.WriteLine("""
|
||||
Usage:
|
||||
dotnet run --project src/Scanner/__Benchmarks/StellaOps.Scanner.Storage.Epss.Perf -c Release -- --rows 310000 --output bench/results/epss-ingest-perf.json
|
||||
|
||||
Options:
|
||||
--rows <int> Row count (default: 310000)
|
||||
--seed <hex> 64-bit seed in hex without 0x (default: 5EED20251219)
|
||||
--model-date <date> Model date (YYYY-MM-DD, default: today)
|
||||
--postgres-image <str> Postgres image (default: postgres:16-alpine)
|
||||
--output <path> Output JSON path (default: bench/results/epss-ingest-perf.json)
|
||||
""");
|
||||
Environment.Exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
if (rowCount < 1)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(rowCount), rowCount, "Row count must be positive.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(postgresImage))
|
||||
{
|
||||
throw new ArgumentException("Postgres image must be provided.", nameof(postgresImage));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(outputPath))
|
||||
{
|
||||
throw new ArgumentException("Output path must be provided.", nameof(outputPath));
|
||||
}
|
||||
|
||||
return new PerfOptions(modelDate, rowCount, seed, postgresImage, outputPath);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record EpssIngestPerfResult
|
||||
{
|
||||
public required PerfToolInfo Tool { get; init; }
|
||||
public required PerfDatasetInfo Dataset { get; init; }
|
||||
public required PerfEnvironmentInfo Environment { get; init; }
|
||||
public required PerfTimingInfo TimingsMs { get; init; }
|
||||
public required PerfWriteResultInfo Result { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record PerfToolInfo
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public required int Schema { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record PerfDatasetInfo
|
||||
{
|
||||
public required string ModelDate { get; init; }
|
||||
public required int Rows { get; init; }
|
||||
public required ulong Seed { get; init; }
|
||||
public required string CompressedSha256 { get; init; }
|
||||
public string? DecompressedSha256 { get; init; }
|
||||
public string? ModelVersionTag { get; init; }
|
||||
public string? PublishedDate { get; init; }
|
||||
public required long CompressedBytes { get; init; }
|
||||
public required long DecompressedBytes { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record PerfEnvironmentInfo
|
||||
{
|
||||
public required string Os { get; init; }
|
||||
public required string Framework { get; init; }
|
||||
public required string ProcessArchitecture { get; init; }
|
||||
public required string PostgresImage { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record PerfTimingInfo
|
||||
{
|
||||
public required long DatasetGenerate { get; init; }
|
||||
public required long ContainerStart { get; init; }
|
||||
public required long Migrations { get; init; }
|
||||
public required long WriteSnapshot { get; init; }
|
||||
public required long Total { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record PerfWriteResultInfo
|
||||
{
|
||||
public required Guid ImportRunId { get; init; }
|
||||
public required int RowCount { get; init; }
|
||||
public required int DistinctCveCount { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
# EPSS Ingest Perf Harness
|
||||
|
||||
Sprint: `SPRINT_3410_0001_0001_epss_ingestion_storage` (Task `EPSS-3410-013A` / `EPSS-3410-014`)
|
||||
|
||||
## Local Run
|
||||
|
||||
Prereqs:
|
||||
- Docker available to Testcontainers
|
||||
- .NET 10 SDK (preview, per repo `global.json`)
|
||||
|
||||
Run (310k rows, default):
|
||||
|
||||
```bash
|
||||
dotnet run --project src/Scanner/__Benchmarks/StellaOps.Scanner.Storage.Epss.Perf/StellaOps.Scanner.Storage.Epss.Perf.csproj -c Release -- --rows 310000 --output bench/results/epss-ingest-perf.json
|
||||
```
|
||||
|
||||
Options:
|
||||
- `--rows <int>`: dataset rows (default: `310000`)
|
||||
- `--seed <hex>`: 64-bit seed in hex without `0x` (default: `5EED20251219`)
|
||||
- `--model-date <YYYY-MM-DD>`: model date (default: today UTC)
|
||||
- `--postgres-image <image>`: Postgres image (default: `postgres:16-alpine`)
|
||||
- `--output <path>`: output JSON path
|
||||
|
||||
## Output Format
|
||||
|
||||
The harness writes a single JSON file:
|
||||
- `tool`: `{ name, schema }`
|
||||
- `dataset`: `{ modelDate, rows, seed, compressedSha256, decompressedSha256, modelVersionTag, publishedDate, compressedBytes, decompressedBytes }`
|
||||
- `environment`: `{ os, framework, processArchitecture, postgresImage }`
|
||||
- `timingsMs`: `{ datasetGenerate, containerStart, migrations, writeSnapshot, total }`
|
||||
- `result`: `{ importRunId, rowCount, distinctCveCount }`
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Testcontainers.PostgreSql" Version="4.1.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.Scanner.Storage\StellaOps.Scanner.Storage.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -119,6 +119,11 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
|
||||
declaredMetadata.Add(new KeyValuePair<string, string?>("lockEditablePathRedacted", "true"));
|
||||
}
|
||||
|
||||
var safeEntry = string.IsNullOrWhiteSpace(entry.EditablePath)
|
||||
? entry
|
||||
: entry with { EditablePath = editableSpec };
|
||||
AppendCommonLockFields(declaredMetadata, safeEntry);
|
||||
|
||||
var componentKey = LanguageExplicitKey.Create("python", "pypi", normalizedName, editableSpec, entry.Locator);
|
||||
writer.AddFromExplicitKey(
|
||||
analyzerId: "python",
|
||||
|
||||
@@ -341,6 +341,18 @@ public sealed class LanguageComponentRecord
|
||||
|
||||
public LanguageComponentSnapshot ToSnapshot()
|
||||
{
|
||||
ComponentThreatVectorSnapshot[]? threatVectors = null;
|
||||
if (_threatVectors.Count > 0)
|
||||
{
|
||||
threatVectors = _threatVectors.Select(static item => new ComponentThreatVectorSnapshot
|
||||
{
|
||||
VectorType = item.VectorType,
|
||||
Confidence = item.Confidence,
|
||||
Evidence = item.Evidence,
|
||||
EntryPath = item.EntryPath,
|
||||
}).ToArray();
|
||||
}
|
||||
|
||||
return new LanguageComponentSnapshot
|
||||
{
|
||||
AnalyzerId = AnalyzerId,
|
||||
@@ -351,14 +363,8 @@ public sealed class LanguageComponentRecord
|
||||
Type = Type,
|
||||
UsedByEntrypoint = UsedByEntrypoint,
|
||||
Intent = Intent,
|
||||
Capabilities = _capabilities.ToArray(),
|
||||
ThreatVectors = _threatVectors.Select(static item => new ComponentThreatVectorSnapshot
|
||||
{
|
||||
VectorType = item.VectorType,
|
||||
Confidence = item.Confidence,
|
||||
Evidence = item.Evidence,
|
||||
EntryPath = item.EntryPath,
|
||||
}).ToArray(),
|
||||
Capabilities = _capabilities.Count == 0 ? null : _capabilities.ToArray(),
|
||||
ThreatVectors = threatVectors,
|
||||
Metadata = _metadata.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal),
|
||||
Evidence = _evidence.Values.Select(static item => new LanguageComponentEvidenceSnapshot
|
||||
{
|
||||
@@ -417,14 +423,14 @@ public sealed class LanguageComponentSnapshot
|
||||
/// </summary>
|
||||
/// <remarks>Part of Sprint 0411 - Semantic Entrypoint Engine (Task 18).</remarks>
|
||||
[JsonPropertyName("capabilities")]
|
||||
public IReadOnlyList<string> Capabilities { get; set; } = Array.Empty<string>();
|
||||
public IReadOnlyList<string>? Capabilities { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Identified threat vectors.
|
||||
/// </summary>
|
||||
/// <remarks>Part of Sprint 0411 - Semantic Entrypoint Engine (Task 18).</remarks>
|
||||
[JsonPropertyName("threatVectors")]
|
||||
public IReadOnlyList<ComponentThreatVectorSnapshot> ThreatVectors { get; set; } = Array.Empty<ComponentThreatVectorSnapshot>();
|
||||
public IReadOnlyList<ComponentThreatVectorSnapshot>? ThreatVectors { get; set; }
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
public IDictionary<string, string?> Metadata { get; set; } = new Dictionary<string, string?>(StringComparer.Ordinal);
|
||||
|
||||
@@ -38,5 +38,9 @@ public static class ScanAnalysisKeys
|
||||
|
||||
public const string DeterminismEvidence = "analysis.determinism.evidence";
|
||||
|
||||
public const string EpssEvidence = "epss.evidence";
|
||||
public const string EpssModelDate = "epss.model_date";
|
||||
public const string EpssNotFoundCves = "epss.not_found";
|
||||
|
||||
public const string ReplaySealedBundleMetadata = "analysis.replay.sealed.bundle";
|
||||
}
|
||||
|
||||
@@ -115,7 +115,8 @@ public sealed class ProofBundleWriter : IProofBundleWriter
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
public ProofBundleWriter(ProofBundleWriterOptions? options = null)
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.EntryTrace;
|
||||
|
||||
/// <summary>
|
||||
/// Outcome classification for entrypoint resolution attempts.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum EntryTraceOutcome
|
||||
{
|
||||
Resolved,
|
||||
@@ -16,6 +18,7 @@ public enum EntryTraceOutcome
|
||||
/// <summary>
|
||||
/// Logical classification for nodes in the entry trace graph.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum EntryTraceNodeKind
|
||||
{
|
||||
Command,
|
||||
@@ -30,6 +33,7 @@ public enum EntryTraceNodeKind
|
||||
/// <summary>
|
||||
/// Interpreter categories supported by the analyzer.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum EntryTraceInterpreterKind
|
||||
{
|
||||
None,
|
||||
@@ -41,6 +45,7 @@ public enum EntryTraceInterpreterKind
|
||||
/// <summary>
|
||||
/// Diagnostic severity levels emitted by the analyzer.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum EntryTraceDiagnosticSeverity
|
||||
{
|
||||
Info,
|
||||
@@ -51,6 +56,7 @@ public enum EntryTraceDiagnosticSeverity
|
||||
/// <summary>
|
||||
/// Enumerates the canonical reasons for unresolved edges.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum EntryTraceUnknownReason
|
||||
{
|
||||
CommandNotFound,
|
||||
@@ -83,6 +89,7 @@ public enum EntryTraceUnknownReason
|
||||
/// <summary>
|
||||
/// Categorises terminal executable kinds.
|
||||
/// </summary>
|
||||
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||
public enum EntryTraceTerminalType
|
||||
{
|
||||
Unknown,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Collections.Frozen;
|
||||
using System.Collections.Immutable;
|
||||
using System.Xml.Linq;
|
||||
|
||||
namespace StellaOps.Scanner.EntryTrace.Semantic.Adapters;
|
||||
|
||||
@@ -175,9 +176,37 @@ public sealed class DotNetSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var framework = (string?)null;
|
||||
|
||||
// Analyze dependencies
|
||||
var packageDependencies = new List<string>();
|
||||
if (context.Dependencies.TryGetValue("dotnet", out var deps))
|
||||
{
|
||||
foreach (var dep in deps)
|
||||
packageDependencies.AddRange(deps);
|
||||
}
|
||||
|
||||
ProjectInfo? projectInfo = null;
|
||||
if (context.ManifestPaths.TryGetValue("project", out var projectPath))
|
||||
{
|
||||
projectInfo = await TryReadProjectInfoAsync(context, projectPath, cancellationToken);
|
||||
if (projectInfo is not null && projectInfo.PackageReferences.Count > 0)
|
||||
{
|
||||
packageDependencies.AddRange(projectInfo.PackageReferences);
|
||||
reasoningChain.Add($"Parsed project file ({projectInfo.PackageReferences.Count} PackageReference)");
|
||||
}
|
||||
|
||||
if (projectInfo?.IsWebSdk == true)
|
||||
{
|
||||
builder.AddCapability(CapabilityClass.NetworkListen);
|
||||
if (intent == ApplicationIntent.Unknown)
|
||||
{
|
||||
intent = ApplicationIntent.WebServer;
|
||||
framework = "aspnetcore";
|
||||
reasoningChain.Add("Project Sdk indicates Web -> WebServer");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (packageDependencies.Count > 0)
|
||||
{
|
||||
foreach (var dep in packageDependencies)
|
||||
{
|
||||
var normalizedDep = NormalizeDependency(dep);
|
||||
|
||||
@@ -186,19 +215,30 @@ public sealed class DotNetSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
if (intent == ApplicationIntent.Unknown || IsHigherPriority(mappedIntent, intent))
|
||||
{
|
||||
intent = mappedIntent;
|
||||
framework = dep;
|
||||
reasoningChain.Add($"Detected {dep} -> {intent}");
|
||||
framework = NormalizeFramework(normalizedDep);
|
||||
reasoningChain.Add($"Detected {normalizedDep} -> {intent}");
|
||||
}
|
||||
|
||||
if (mappedIntent is ApplicationIntent.WebServer or ApplicationIntent.RpcServer or ApplicationIntent.GraphQlServer)
|
||||
builder.AddCapability(CapabilityClass.NetworkListen);
|
||||
else if (mappedIntent is ApplicationIntent.Worker or ApplicationIntent.StreamProcessor)
|
||||
builder.AddCapability(CapabilityClass.MessageQueue);
|
||||
}
|
||||
|
||||
if (PackageCapabilityMap.TryGetValue(normalizedDep, out var capability))
|
||||
{
|
||||
builder.AddCapability(capability);
|
||||
reasoningChain.Add($"Package {dep} -> {capability}");
|
||||
reasoningChain.Add($"Package {normalizedDep} -> {capability}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (intent == ApplicationIntent.Unknown && projectInfo?.OutputTypeExe == true)
|
||||
{
|
||||
intent = ApplicationIntent.CliTool;
|
||||
reasoningChain.Add("Project OutputType=Exe -> CliTool");
|
||||
}
|
||||
|
||||
// Analyze entrypoint command
|
||||
var cmdSignals = AnalyzeCommand(context.Specification);
|
||||
if (cmdSignals.Intent != ApplicationIntent.Unknown && intent == ApplicationIntent.Unknown)
|
||||
@@ -262,6 +302,17 @@ public sealed class DotNetSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
return parts[0].Trim();
|
||||
}
|
||||
|
||||
private static string? NormalizeFramework(string normalizedDependency)
|
||||
{
|
||||
if (normalizedDependency.StartsWith("Microsoft.AspNetCore", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(normalizedDependency, "Swashbuckle.AspNetCore", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return "aspnetcore";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsHigherPriority(ApplicationIntent newer, ApplicationIntent current)
|
||||
{
|
||||
var priorityOrder = new[]
|
||||
@@ -358,4 +409,61 @@ public sealed class DotNetSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var hash = context.ImageDigest ?? Guid.NewGuid().ToString("N");
|
||||
return $"sem-dotnet-{hash[..12]}";
|
||||
}
|
||||
|
||||
private sealed record ProjectInfo(
|
||||
bool IsWebSdk,
|
||||
bool OutputTypeExe,
|
||||
IReadOnlyList<string> PackageReferences);
|
||||
|
||||
private static async Task<ProjectInfo?> TryReadProjectInfoAsync(
|
||||
SemanticAnalysisContext context,
|
||||
string projectPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var content = await context.FileSystem.TryReadFileAsync(projectPath, cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var doc = XDocument.Parse(content);
|
||||
var root = doc.Root;
|
||||
|
||||
var sdk = root?.Attribute("Sdk")?.Value?.Trim();
|
||||
var isWebSdk = !string.IsNullOrWhiteSpace(sdk) &&
|
||||
sdk.Contains("Microsoft.NET.Sdk.Web", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
var outputType = doc.Descendants()
|
||||
.FirstOrDefault(element => element.Name.LocalName == "OutputType")
|
||||
?.Value
|
||||
?.Trim();
|
||||
|
||||
var outputTypeExe = string.Equals(outputType, "Exe", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
var packageReferences = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var element in doc.Descendants().Where(element => element.Name.LocalName == "PackageReference"))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var include = element.Attribute("Include")?.Value?.Trim();
|
||||
var update = element.Attribute("Update")?.Value?.Trim();
|
||||
var name = !string.IsNullOrWhiteSpace(include) ? include : update;
|
||||
if (!string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
packageReferences.Add(name);
|
||||
}
|
||||
}
|
||||
|
||||
return new ProjectInfo(
|
||||
IsWebSdk: isWebSdk,
|
||||
OutputTypeExe: outputTypeExe,
|
||||
PackageReferences: packageReferences.OrderBy(static name => name, StringComparer.Ordinal).ToArray());
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Collections.Frozen;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.EntryTrace.Semantic.Adapters;
|
||||
|
||||
@@ -192,9 +193,31 @@ public sealed class GoSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var framework = (string?)null;
|
||||
|
||||
// Analyze dependencies (go.mod imports)
|
||||
var moduleDependencies = new List<string>();
|
||||
if (context.Dependencies.TryGetValue("go", out var deps))
|
||||
{
|
||||
foreach (var dep in deps)
|
||||
moduleDependencies.AddRange(deps);
|
||||
}
|
||||
|
||||
if (context.ManifestPaths.TryGetValue("go.mod", out var goModPath))
|
||||
{
|
||||
var goModDependencies = await TryReadGoModDependenciesAsync(context, goModPath, cancellationToken);
|
||||
if (goModDependencies.Count > 0)
|
||||
{
|
||||
moduleDependencies.AddRange(goModDependencies);
|
||||
reasoningChain.Add($"Parsed go.mod ({goModDependencies.Count} deps)");
|
||||
}
|
||||
|
||||
if (await DetectNetHttpUsageAsync(context, goModPath, cancellationToken))
|
||||
{
|
||||
moduleDependencies.Add("net/http");
|
||||
reasoningChain.Add("Detected net/http usage in source");
|
||||
}
|
||||
}
|
||||
|
||||
if (moduleDependencies.Count > 0)
|
||||
{
|
||||
foreach (var dep in moduleDependencies)
|
||||
{
|
||||
var normalizedDep = NormalizeDependency(dep);
|
||||
|
||||
@@ -203,15 +226,20 @@ public sealed class GoSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
if (intent == ApplicationIntent.Unknown || IsHigherPriority(mappedIntent, intent))
|
||||
{
|
||||
intent = mappedIntent;
|
||||
framework = dep;
|
||||
reasoningChain.Add($"Detected {dep} -> {intent}");
|
||||
framework = normalizedDep;
|
||||
reasoningChain.Add($"Detected {normalizedDep} -> {intent}");
|
||||
}
|
||||
|
||||
if (mappedIntent is ApplicationIntent.WebServer or ApplicationIntent.RpcServer or ApplicationIntent.GraphQlServer)
|
||||
builder.AddCapability(CapabilityClass.NetworkListen);
|
||||
else if (mappedIntent is ApplicationIntent.Worker or ApplicationIntent.StreamProcessor or ApplicationIntent.MessageBroker)
|
||||
builder.AddCapability(CapabilityClass.MessageQueue);
|
||||
}
|
||||
|
||||
if (ModuleCapabilityMap.TryGetValue(normalizedDep, out var capability))
|
||||
{
|
||||
builder.AddCapability(capability);
|
||||
reasoningChain.Add($"Module {dep} -> {capability}");
|
||||
reasoningChain.Add($"Module {normalizedDep} -> {capability}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -263,9 +291,20 @@ public sealed class GoSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
|
||||
private static string NormalizeDependency(string dep)
|
||||
{
|
||||
// Handle Go module paths with versions
|
||||
var parts = dep.Split('@');
|
||||
return parts[0].Trim();
|
||||
// Handle Go module paths with versions (both @ and whitespace forms):
|
||||
// - github.com/spf13/cobra@v1.7.0 -> github.com/spf13/cobra
|
||||
// - github.com/spf13/cobra v1.7.0 -> github.com/spf13/cobra
|
||||
var trimmed = dep.Trim();
|
||||
if (trimmed.Length == 0)
|
||||
{
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
var whitespaceParts = trimmed.Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries);
|
||||
trimmed = whitespaceParts.Length > 0 ? whitespaceParts[0] : trimmed;
|
||||
|
||||
var atParts = trimmed.Split('@');
|
||||
return atParts[0].Trim();
|
||||
}
|
||||
|
||||
private static bool IsHigherPriority(ApplicationIntent newer, ApplicationIntent current)
|
||||
@@ -367,4 +406,120 @@ public sealed class GoSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var hash = context.ImageDigest ?? Guid.NewGuid().ToString("N");
|
||||
return $"sem-go-{hash[..12]}";
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<string>> TryReadGoModDependenciesAsync(
|
||||
SemanticAnalysisContext context,
|
||||
string goModPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var content = await context.FileSystem.TryReadFileAsync(goModPath, cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var dependencies = new HashSet<string>(StringComparer.Ordinal);
|
||||
using var reader = new StringReader(content);
|
||||
string? line;
|
||||
var inRequireBlock = false;
|
||||
|
||||
while ((line = reader.ReadLine()) is not null)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var trimmed = line.Trim();
|
||||
if (trimmed.Length == 0 || trimmed.StartsWith("//", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (inRequireBlock)
|
||||
{
|
||||
if (trimmed == ")")
|
||||
{
|
||||
inRequireBlock = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
var parts = trimmed.Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length > 0)
|
||||
{
|
||||
dependencies.Add(parts[0]);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (trimmed.StartsWith("require (", StringComparison.Ordinal))
|
||||
{
|
||||
inRequireBlock = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (trimmed.StartsWith("require ", StringComparison.Ordinal))
|
||||
{
|
||||
var rest = trimmed["require ".Length..].Trim();
|
||||
var parts = rest.Split([' ', '\t'], StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length > 0)
|
||||
{
|
||||
dependencies.Add(parts[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (dependencies.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return dependencies.OrderBy(static dependency => dependency, StringComparer.Ordinal).ToArray();
|
||||
}
|
||||
|
||||
private static async Task<bool> DetectNetHttpUsageAsync(
|
||||
SemanticAnalysisContext context,
|
||||
string goModPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var directory = GetDirectory(goModPath);
|
||||
if (directory is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var goFiles = await context.FileSystem.ListFilesAsync(directory, "*.go", cancellationToken);
|
||||
foreach (var file in goFiles)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var content = await context.FileSystem.TryReadFileAsync(file, cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (content.Contains("net/http", StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string? GetDirectory(string path)
|
||||
{
|
||||
var normalized = path.Replace('\\', '/');
|
||||
var lastSlash = normalized.LastIndexOf('/');
|
||||
if (lastSlash < 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (lastSlash == 0)
|
||||
{
|
||||
return "/";
|
||||
}
|
||||
|
||||
return normalized[..lastSlash];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Collections.Frozen;
|
||||
using System.Collections.Immutable;
|
||||
using System.Xml.Linq;
|
||||
|
||||
namespace StellaOps.Scanner.EntryTrace.Semantic.Adapters;
|
||||
|
||||
@@ -183,9 +184,25 @@ public sealed class JavaSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var framework = (string?)null;
|
||||
|
||||
// Analyze dependencies
|
||||
var javaDependencies = new List<string>();
|
||||
if (context.Dependencies.TryGetValue("java", out var deps))
|
||||
{
|
||||
foreach (var dep in deps)
|
||||
javaDependencies.AddRange(deps);
|
||||
}
|
||||
|
||||
if (context.ManifestPaths.TryGetValue("pom.xml", out var pomPath))
|
||||
{
|
||||
var pomDependencies = await TryReadPomDependenciesAsync(context, pomPath, cancellationToken);
|
||||
if (pomDependencies.Count > 0)
|
||||
{
|
||||
javaDependencies.AddRange(pomDependencies);
|
||||
reasoningChain.Add($"Parsed pom.xml ({pomDependencies.Count} deps)");
|
||||
}
|
||||
}
|
||||
|
||||
if (javaDependencies.Count > 0)
|
||||
{
|
||||
foreach (var dep in javaDependencies)
|
||||
{
|
||||
var normalizedDep = NormalizeDependency(dep);
|
||||
|
||||
@@ -194,15 +211,20 @@ public sealed class JavaSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
if (intent == ApplicationIntent.Unknown || IsHigherPriority(mappedIntent, intent))
|
||||
{
|
||||
intent = mappedIntent;
|
||||
framework = dep;
|
||||
reasoningChain.Add($"Detected {dep} -> {intent}");
|
||||
framework = normalizedDep;
|
||||
reasoningChain.Add($"Detected {normalizedDep} -> {intent}");
|
||||
}
|
||||
|
||||
if (mappedIntent == ApplicationIntent.WebServer)
|
||||
builder.AddCapability(CapabilityClass.NetworkListen);
|
||||
else if (mappedIntent is ApplicationIntent.Worker or ApplicationIntent.StreamProcessor)
|
||||
builder.AddCapability(CapabilityClass.MessageQueue);
|
||||
}
|
||||
|
||||
if (DependencyCapabilityMap.TryGetValue(normalizedDep, out var capability))
|
||||
{
|
||||
builder.AddCapability(capability);
|
||||
reasoningChain.Add($"Dependency {dep} -> {capability}");
|
||||
reasoningChain.Add($"Dependency {normalizedDep} -> {capability}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -367,4 +389,59 @@ public sealed class JavaSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var hash = context.ImageDigest ?? Guid.NewGuid().ToString("N");
|
||||
return $"sem-java-{hash[..12]}";
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<string>> TryReadPomDependenciesAsync(
|
||||
SemanticAnalysisContext context,
|
||||
string pomPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var content = await context.FileSystem.TryReadFileAsync(pomPath, cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var tokens = new HashSet<string>(StringComparer.Ordinal);
|
||||
var doc = XDocument.Parse(content);
|
||||
|
||||
foreach (var dep in doc.Descendants().Where(element => element.Name.LocalName == "dependency"))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var groupId = dep.Elements().FirstOrDefault(element => element.Name.LocalName == "groupId")?.Value?.Trim();
|
||||
var artifactId = dep.Elements().FirstOrDefault(element => element.Name.LocalName == "artifactId")?.Value?.Trim();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(groupId))
|
||||
{
|
||||
if (groupId.StartsWith("org.springframework.boot", StringComparison.Ordinal))
|
||||
{
|
||||
tokens.Add("spring-boot");
|
||||
}
|
||||
|
||||
if (groupId.StartsWith("io.quarkus", StringComparison.Ordinal))
|
||||
{
|
||||
tokens.Add("quarkus");
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(artifactId))
|
||||
{
|
||||
tokens.Add(artifactId.ToLowerInvariant().Replace("_", "-"));
|
||||
}
|
||||
}
|
||||
|
||||
if (tokens.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return tokens.OrderBy(static token => token, StringComparer.Ordinal).ToArray();
|
||||
}
|
||||
catch
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Collections.Frozen;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Scanner.EntryTrace.Semantic.Adapters;
|
||||
|
||||
@@ -209,9 +210,29 @@ public sealed class NodeSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var framework = (string?)null;
|
||||
|
||||
// Analyze dependencies
|
||||
if (context.Dependencies.TryGetValue("node", out var deps))
|
||||
context.ManifestPaths.TryGetValue("package.json", out var packageJsonPath);
|
||||
|
||||
var nodeDependencies = new List<string>();
|
||||
if (context.Dependencies.TryGetValue("node", out var deps) ||
|
||||
context.Dependencies.TryGetValue("javascript", out deps) ||
|
||||
context.Dependencies.TryGetValue("typescript", out deps))
|
||||
{
|
||||
foreach (var dep in deps)
|
||||
nodeDependencies.AddRange(deps);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(packageJsonPath))
|
||||
{
|
||||
var manifestDependencies = await TryReadPackageJsonDependenciesAsync(context, packageJsonPath, cancellationToken);
|
||||
if (manifestDependencies.Count > 0)
|
||||
{
|
||||
nodeDependencies.AddRange(manifestDependencies);
|
||||
reasoningChain.Add($"Parsed package.json ({manifestDependencies.Count} deps)");
|
||||
}
|
||||
}
|
||||
|
||||
if (nodeDependencies.Count > 0)
|
||||
{
|
||||
foreach (var dep in nodeDependencies)
|
||||
{
|
||||
var normalizedDep = NormalizeDependency(dep);
|
||||
|
||||
@@ -220,19 +241,31 @@ public sealed class NodeSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
if (intent == ApplicationIntent.Unknown || IsHigherPriority(mappedIntent, intent))
|
||||
{
|
||||
intent = mappedIntent;
|
||||
framework = dep;
|
||||
reasoningChain.Add($"Detected {dep} -> {intent}");
|
||||
framework = NormalizeFramework(normalizedDep);
|
||||
reasoningChain.Add($"Detected {normalizedDep} -> {intent}");
|
||||
}
|
||||
|
||||
if (mappedIntent is ApplicationIntent.WebServer or ApplicationIntent.RpcServer or ApplicationIntent.GraphQlServer)
|
||||
builder.AddCapability(CapabilityClass.NetworkListen);
|
||||
else if (mappedIntent is ApplicationIntent.Worker or ApplicationIntent.StreamProcessor)
|
||||
builder.AddCapability(CapabilityClass.MessageQueue);
|
||||
}
|
||||
|
||||
if (PackageCapabilityMap.TryGetValue(normalizedDep, out var capability))
|
||||
{
|
||||
builder.AddCapability(capability);
|
||||
reasoningChain.Add($"Package {dep} -> {capability}");
|
||||
reasoningChain.Add($"Package {normalizedDep} -> {capability}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Serverless manifest hint (e.g., serverless.yml discovered by earlier filesystem pass).
|
||||
if (intent == ApplicationIntent.Unknown && context.ManifestPaths.ContainsKey("serverless"))
|
||||
{
|
||||
intent = ApplicationIntent.Serverless;
|
||||
reasoningChain.Add("Manifest hint: serverless -> Serverless");
|
||||
}
|
||||
|
||||
// Analyze entrypoint command
|
||||
var cmdSignals = AnalyzeCommand(context.Specification);
|
||||
if (cmdSignals.Intent != ApplicationIntent.Unknown && intent == ApplicationIntent.Unknown)
|
||||
@@ -247,9 +280,9 @@ public sealed class NodeSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
}
|
||||
|
||||
// Check package.json for bin entries -> CLI tool
|
||||
if (context.ManifestPaths.TryGetValue("package.json", out var pkgPath))
|
||||
if (!string.IsNullOrWhiteSpace(packageJsonPath))
|
||||
{
|
||||
if (await HasBinEntriesAsync(context, pkgPath, cancellationToken))
|
||||
if (await HasBinEntriesAsync(context, packageJsonPath, cancellationToken))
|
||||
{
|
||||
if (intent == ApplicationIntent.Unknown)
|
||||
{
|
||||
@@ -286,10 +319,87 @@ public sealed class NodeSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
|
||||
private static string NormalizeDependency(string dep)
|
||||
{
|
||||
// Handle scoped packages and versions
|
||||
return dep.ToLowerInvariant()
|
||||
.Split('@')[0] // Remove version
|
||||
.Trim();
|
||||
// Handle scoped packages and versions:
|
||||
// - express@4.18.0 -> express
|
||||
// - @nestjs/core -> @nestjs/core
|
||||
// - @nestjs/core@10.0.0 -> @nestjs/core
|
||||
var normalized = dep.Trim().ToLowerInvariant();
|
||||
if (normalized.Length == 0)
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
if (normalized.StartsWith("@", StringComparison.Ordinal))
|
||||
{
|
||||
var lastAt = normalized.LastIndexOf('@');
|
||||
return lastAt > 0 ? normalized[..lastAt] : normalized;
|
||||
}
|
||||
|
||||
var at = normalized.IndexOf('@', StringComparison.Ordinal);
|
||||
return at > 0 ? normalized[..at] : normalized;
|
||||
}
|
||||
|
||||
private static string NormalizeFramework(string normalizedDependency)
|
||||
{
|
||||
return normalizedDependency switch
|
||||
{
|
||||
"nest" or "@nestjs/core" or "@nestjs/platform-express" => "nestjs",
|
||||
_ => normalizedDependency
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<string>> TryReadPackageJsonDependenciesAsync(
|
||||
SemanticAnalysisContext context,
|
||||
string pkgPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var content = await context.FileSystem.TryReadFileAsync(pkgPath, cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(content);
|
||||
if (doc.RootElement.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var dependencies = new HashSet<string>(StringComparer.Ordinal);
|
||||
AddDependencyObjectKeys(doc.RootElement, "dependencies", dependencies);
|
||||
AddDependencyObjectKeys(doc.RootElement, "devDependencies", dependencies);
|
||||
AddDependencyObjectKeys(doc.RootElement, "peerDependencies", dependencies);
|
||||
AddDependencyObjectKeys(doc.RootElement, "optionalDependencies", dependencies);
|
||||
|
||||
if (dependencies.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return dependencies.OrderBy(static dep => dep, StringComparer.Ordinal).ToArray();
|
||||
}
|
||||
catch
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
}
|
||||
|
||||
private static void AddDependencyObjectKeys(JsonElement root, string propertyName, HashSet<string> dependencies)
|
||||
{
|
||||
if (!root.TryGetProperty(propertyName, out var section) || section.ValueKind != JsonValueKind.Object)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var property in section.EnumerateObject())
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(property.Name))
|
||||
{
|
||||
dependencies.Add(property.Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsHigherPriority(ApplicationIntent newer, ApplicationIntent current)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Collections.Frozen;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.EntryTrace.Semantic.Adapters;
|
||||
|
||||
@@ -188,9 +189,29 @@ public sealed class PythonSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var framework = (string?)null;
|
||||
|
||||
// Analyze dependencies to determine intent and capabilities
|
||||
var pythonDependencies = new List<string>();
|
||||
if (context.Dependencies.TryGetValue("python", out var deps))
|
||||
{
|
||||
foreach (var dep in deps)
|
||||
pythonDependencies.AddRange(deps);
|
||||
}
|
||||
else
|
||||
{
|
||||
pythonDependencies = [];
|
||||
}
|
||||
|
||||
if (pythonDependencies.Count == 0)
|
||||
{
|
||||
var requirementsDeps = await TryReadRequirementsDependenciesAsync(context, cancellationToken);
|
||||
if (requirementsDeps.Count > 0)
|
||||
{
|
||||
pythonDependencies.AddRange(requirementsDeps);
|
||||
reasoningChain.Add($"Parsed requirements.txt ({requirementsDeps.Count} deps)");
|
||||
}
|
||||
}
|
||||
|
||||
if (pythonDependencies.Count > 0)
|
||||
{
|
||||
foreach (var dep in pythonDependencies)
|
||||
{
|
||||
var normalizedDep = NormalizeDependency(dep);
|
||||
|
||||
@@ -200,20 +221,33 @@ public sealed class PythonSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
if (intent == ApplicationIntent.Unknown || IsHigherPriority(mappedIntent, intent))
|
||||
{
|
||||
intent = mappedIntent;
|
||||
framework = dep;
|
||||
reasoningChain.Add($"Detected {dep} -> {intent}");
|
||||
framework = normalizedDep;
|
||||
reasoningChain.Add($"Detected {normalizedDep} -> {intent}");
|
||||
}
|
||||
|
||||
// Baseline capabilities implied by the inferred intent/framework.
|
||||
if (mappedIntent == ApplicationIntent.WebServer)
|
||||
builder.AddCapability(CapabilityClass.NetworkListen);
|
||||
else if (mappedIntent is ApplicationIntent.Worker or ApplicationIntent.StreamProcessor)
|
||||
builder.AddCapability(CapabilityClass.MessageQueue);
|
||||
}
|
||||
|
||||
// Check capability imports
|
||||
if (ImportCapabilityMap.TryGetValue(normalizedDep, out var capability))
|
||||
{
|
||||
builder.AddCapability(capability);
|
||||
reasoningChain.Add($"Import {dep} -> {capability}");
|
||||
reasoningChain.Add($"Import {normalizedDep} -> {capability}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Serverless manifest hint (e.g., Serverless Framework / SAM markers discovered earlier in the scan).
|
||||
if (intent == ApplicationIntent.Unknown && context.ManifestPaths.ContainsKey("serverless"))
|
||||
{
|
||||
intent = ApplicationIntent.Serverless;
|
||||
reasoningChain.Add("Manifest hint: serverless -> Serverless");
|
||||
}
|
||||
|
||||
// Analyze entrypoint command for additional signals
|
||||
var cmdSignals = AnalyzeCommand(context.Specification);
|
||||
if (cmdSignals.Intent != ApplicationIntent.Unknown && intent == ApplicationIntent.Unknown)
|
||||
@@ -353,4 +387,87 @@ public sealed class PythonSemanticAdapter : ISemanticEntrypointAnalyzer
|
||||
var hash = context.ImageDigest ?? Guid.NewGuid().ToString("N");
|
||||
return $"sem-py-{hash[..12]}";
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<string>> TryReadRequirementsDependenciesAsync(
|
||||
SemanticAnalysisContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var entrypoint = context.Specification.Entrypoint.FirstOrDefault();
|
||||
if (string.IsNullOrWhiteSpace(entrypoint) || !entrypoint.Contains('/', StringComparison.Ordinal))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var directory = GetDirectory(entrypoint);
|
||||
if (directory is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var candidate = directory == "/" ? "/requirements.txt" : $"{directory}/requirements.txt";
|
||||
var content = await context.FileSystem.TryReadFileAsync(candidate, cancellationToken);
|
||||
if (string.IsNullOrWhiteSpace(content))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var dependencies = new HashSet<string>(StringComparer.Ordinal);
|
||||
using var reader = new StringReader(content);
|
||||
string? line;
|
||||
while ((line = reader.ReadLine()) is not null)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var trimmed = line.Trim();
|
||||
if (trimmed.Length == 0 || trimmed.StartsWith("#", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var commentIndex = trimmed.IndexOf('#');
|
||||
if (commentIndex >= 0)
|
||||
{
|
||||
trimmed = trimmed[..commentIndex].Trim();
|
||||
if (trimmed.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (trimmed.StartsWith("-", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var normalized = NormalizeDependency(trimmed);
|
||||
if (!string.IsNullOrWhiteSpace(normalized))
|
||||
{
|
||||
dependencies.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
if (dependencies.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return dependencies.OrderBy(static dep => dep, StringComparer.Ordinal).ToArray();
|
||||
}
|
||||
|
||||
private static string? GetDirectory(string path)
|
||||
{
|
||||
var normalized = path.Replace('\\', '/');
|
||||
var lastSlash = normalized.LastIndexOf('/');
|
||||
if (lastSlash < 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (lastSlash == 0)
|
||||
{
|
||||
return "/";
|
||||
}
|
||||
|
||||
return normalized[..lastSlash];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,58 +13,58 @@
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Check if table exists
|
||||
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'vuln_instance_triage') THEN
|
||||
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'vuln_instance_triage') THEN
|
||||
-- Add current_epss_score column
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'vuln_instance_triage' AND column_name = 'current_epss_score') THEN
|
||||
WHERE table_schema = current_schema() AND table_name = 'vuln_instance_triage' AND column_name = 'current_epss_score') THEN
|
||||
ALTER TABLE vuln_instance_triage ADD COLUMN current_epss_score DOUBLE PRECISION;
|
||||
COMMENT ON COLUMN vuln_instance_triage.current_epss_score IS 'Current EPSS probability score [0,1]';
|
||||
END IF;
|
||||
|
||||
-- Add current_epss_percentile column
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'vuln_instance_triage' AND column_name = 'current_epss_percentile') THEN
|
||||
WHERE table_schema = current_schema() AND table_name = 'vuln_instance_triage' AND column_name = 'current_epss_percentile') THEN
|
||||
ALTER TABLE vuln_instance_triage ADD COLUMN current_epss_percentile DOUBLE PRECISION;
|
||||
COMMENT ON COLUMN vuln_instance_triage.current_epss_percentile IS 'Current EPSS percentile rank [0,1]';
|
||||
END IF;
|
||||
|
||||
-- Add current_epss_band column
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'vuln_instance_triage' AND column_name = 'current_epss_band') THEN
|
||||
WHERE table_schema = current_schema() AND table_name = 'vuln_instance_triage' AND column_name = 'current_epss_band') THEN
|
||||
ALTER TABLE vuln_instance_triage ADD COLUMN current_epss_band TEXT;
|
||||
COMMENT ON COLUMN vuln_instance_triage.current_epss_band IS 'Current EPSS priority band: CRITICAL, HIGH, MEDIUM, LOW';
|
||||
END IF;
|
||||
|
||||
-- Add epss_model_date column
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'vuln_instance_triage' AND column_name = 'epss_model_date') THEN
|
||||
WHERE table_schema = current_schema() AND table_name = 'vuln_instance_triage' AND column_name = 'epss_model_date') THEN
|
||||
ALTER TABLE vuln_instance_triage ADD COLUMN epss_model_date DATE;
|
||||
COMMENT ON COLUMN vuln_instance_triage.epss_model_date IS 'EPSS model date when last updated';
|
||||
END IF;
|
||||
|
||||
-- Add epss_updated_at column
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'vuln_instance_triage' AND column_name = 'epss_updated_at') THEN
|
||||
WHERE table_schema = current_schema() AND table_name = 'vuln_instance_triage' AND column_name = 'epss_updated_at') THEN
|
||||
ALTER TABLE vuln_instance_triage ADD COLUMN epss_updated_at TIMESTAMPTZ;
|
||||
COMMENT ON COLUMN vuln_instance_triage.epss_updated_at IS 'Timestamp when EPSS data was last updated';
|
||||
END IF;
|
||||
|
||||
-- Add previous_epss_band column (for change tracking)
|
||||
IF NOT EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'vuln_instance_triage' AND column_name = 'previous_epss_band') THEN
|
||||
WHERE table_schema = current_schema() AND table_name = 'vuln_instance_triage' AND column_name = 'previous_epss_band') THEN
|
||||
ALTER TABLE vuln_instance_triage ADD COLUMN previous_epss_band TEXT;
|
||||
COMMENT ON COLUMN vuln_instance_triage.previous_epss_band IS 'Previous EPSS priority band before last update';
|
||||
END IF;
|
||||
|
||||
-- Create index for efficient band-based queries
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_vuln_instance_epss_band') THEN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = current_schema() AND indexname = 'idx_vuln_instance_epss_band') THEN
|
||||
CREATE INDEX idx_vuln_instance_epss_band
|
||||
ON vuln_instance_triage (current_epss_band)
|
||||
WHERE current_epss_band IN ('CRITICAL', 'HIGH');
|
||||
END IF;
|
||||
|
||||
-- Create index for stale EPSS data detection
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_vuln_instance_epss_model_date') THEN
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = current_schema() AND indexname = 'idx_vuln_instance_epss_model_date') THEN
|
||||
CREATE INDEX idx_vuln_instance_epss_model_date
|
||||
ON vuln_instance_triage (epss_model_date);
|
||||
END IF;
|
||||
@@ -80,6 +80,10 @@ END $$;
|
||||
-- ============================================================================
|
||||
-- Efficiently updates EPSS data for multiple vulnerability instances
|
||||
|
||||
DO $epss_triage$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = 'vuln_instance_triage') THEN
|
||||
EXECUTE $sql$
|
||||
CREATE OR REPLACE FUNCTION batch_update_epss_triage(
|
||||
p_updates JSONB,
|
||||
p_model_date DATE,
|
||||
@@ -127,14 +131,13 @@ BEGIN
|
||||
RETURN QUERY SELECT v_updated, v_band_changes;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
$sql$;
|
||||
|
||||
EXECUTE $sql$
|
||||
COMMENT ON FUNCTION batch_update_epss_triage IS 'Batch updates EPSS data for vulnerability instances, tracking band changes';
|
||||
$sql$;
|
||||
|
||||
-- ============================================================================
|
||||
-- View for Instances Needing EPSS Update
|
||||
-- ============================================================================
|
||||
-- Returns instances with stale or missing EPSS data
|
||||
|
||||
EXECUTE $sql$
|
||||
CREATE OR REPLACE VIEW v_epss_stale_instances AS
|
||||
SELECT
|
||||
vit.instance_id,
|
||||
@@ -146,5 +149,12 @@ SELECT
|
||||
FROM vuln_instance_triage vit
|
||||
WHERE vit.epss_model_date IS NULL
|
||||
OR vit.epss_model_date < CURRENT_DATE - 1;
|
||||
$sql$;
|
||||
|
||||
EXECUTE $sql$
|
||||
COMMENT ON VIEW v_epss_stale_instances IS 'Instances with stale or missing EPSS data, needing enrichment';
|
||||
$sql$;
|
||||
ELSE
|
||||
RAISE NOTICE 'Table vuln_instance_triage does not exist; skipping EPSS triage function/view';
|
||||
END IF;
|
||||
END $epss_triage$;
|
||||
|
||||
@@ -3,23 +3,17 @@
|
||||
-- Sprint: SPRINT_3700_0002_0001_vuln_surfaces_core
|
||||
-- Task: SURF-014
|
||||
-- Description: Vulnerability surface storage for trigger method analysis.
|
||||
--
|
||||
-- Note: migrations are executed with the module schema as the active search_path.
|
||||
-- Keep objects unqualified so integration tests can run in isolated schemas.
|
||||
-- =============================================================================
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- Prevent re-running
|
||||
DO $$ BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'scanner' AND tablename = 'vuln_surfaces') THEN
|
||||
RAISE EXCEPTION 'Migration 014_vuln_surfaces already applied';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- =============================================================================
|
||||
-- VULN_SURFACES: Computed vulnerability surface for CVE + package + version
|
||||
-- =============================================================================
|
||||
CREATE TABLE scanner.vuln_surfaces (
|
||||
CREATE TABLE IF NOT EXISTS vuln_surfaces (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tenant_id UUID NOT NULL REFERENCES public.tenants(id),
|
||||
tenant_id UUID NOT NULL,
|
||||
|
||||
-- CVE/vulnerability identity
|
||||
cve_id TEXT NOT NULL,
|
||||
@@ -41,23 +35,22 @@ CREATE TABLE scanner.vuln_surfaces (
|
||||
-- DSSE attestation (optional)
|
||||
attestation_digest TEXT,
|
||||
|
||||
-- Indexes for lookups
|
||||
CONSTRAINT uq_vuln_surface_key UNIQUE (tenant_id, cve_id, package_ecosystem, package_name, vuln_version)
|
||||
);
|
||||
|
||||
-- Indexes for common queries
|
||||
CREATE INDEX idx_vuln_surfaces_cve ON scanner.vuln_surfaces(tenant_id, cve_id);
|
||||
CREATE INDEX idx_vuln_surfaces_package ON scanner.vuln_surfaces(tenant_id, package_ecosystem, package_name);
|
||||
CREATE INDEX idx_vuln_surfaces_computed_at ON scanner.vuln_surfaces(computed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surfaces_cve ON vuln_surfaces(tenant_id, cve_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surfaces_package ON vuln_surfaces(tenant_id, package_ecosystem, package_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surfaces_computed_at ON vuln_surfaces(computed_at DESC);
|
||||
|
||||
COMMENT ON TABLE scanner.vuln_surfaces IS 'Computed vulnerability surfaces identifying which methods changed between vulnerable and fixed versions';
|
||||
COMMENT ON TABLE vuln_surfaces IS 'Computed vulnerability surfaces identifying which methods changed between vulnerable and fixed versions';
|
||||
|
||||
-- =============================================================================
|
||||
-- VULN_SURFACE_SINKS: Individual trigger methods for a vulnerability surface
|
||||
-- =============================================================================
|
||||
CREATE TABLE scanner.vuln_surface_sinks (
|
||||
CREATE TABLE IF NOT EXISTS vuln_surface_sinks (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
surface_id UUID NOT NULL REFERENCES scanner.vuln_surfaces(id) ON DELETE CASCADE,
|
||||
surface_id UUID NOT NULL REFERENCES vuln_surfaces(id) ON DELETE CASCADE,
|
||||
|
||||
-- Method identity
|
||||
method_key TEXT NOT NULL, -- Normalized method signature (FQN)
|
||||
@@ -82,24 +75,23 @@ CREATE TABLE scanner.vuln_surface_sinks (
|
||||
start_line INTEGER,
|
||||
end_line INTEGER,
|
||||
|
||||
-- Indexes for lookups
|
||||
CONSTRAINT uq_surface_sink_key UNIQUE (surface_id, method_key)
|
||||
);
|
||||
|
||||
-- Indexes for common queries
|
||||
CREATE INDEX idx_vuln_surface_sinks_surface ON scanner.vuln_surface_sinks(surface_id);
|
||||
CREATE INDEX idx_vuln_surface_sinks_method ON scanner.vuln_surface_sinks(method_name);
|
||||
CREATE INDEX idx_vuln_surface_sinks_type ON scanner.vuln_surface_sinks(declaring_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surface_sinks_surface ON vuln_surface_sinks(surface_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surface_sinks_method ON vuln_surface_sinks(method_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surface_sinks_type ON vuln_surface_sinks(declaring_type);
|
||||
|
||||
COMMENT ON TABLE scanner.vuln_surface_sinks IS 'Individual methods that changed between vulnerable and fixed package versions';
|
||||
COMMENT ON TABLE vuln_surface_sinks IS 'Individual methods that changed between vulnerable and fixed package versions';
|
||||
|
||||
-- =============================================================================
|
||||
-- VULN_SURFACE_TRIGGERS: Links sinks to call graph nodes where they are invoked
|
||||
-- =============================================================================
|
||||
CREATE TABLE scanner.vuln_surface_triggers (
|
||||
CREATE TABLE IF NOT EXISTS vuln_surface_triggers (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
sink_id UUID NOT NULL REFERENCES scanner.vuln_surface_sinks(id) ON DELETE CASCADE,
|
||||
scan_id UUID NOT NULL, -- References scanner.scans
|
||||
sink_id UUID NOT NULL REFERENCES vuln_surface_sinks(id) ON DELETE CASCADE,
|
||||
scan_id UUID NOT NULL, -- References scans.scan_id
|
||||
|
||||
-- Caller identity
|
||||
caller_node_id TEXT NOT NULL, -- Call graph node ID
|
||||
@@ -116,34 +108,33 @@ CREATE TABLE scanner.vuln_surface_triggers (
|
||||
call_type TEXT NOT NULL DEFAULT 'direct', -- 'direct', 'virtual', 'interface', 'reflection'
|
||||
is_conditional BOOLEAN NOT NULL DEFAULT false,
|
||||
|
||||
-- Indexes for lookups
|
||||
CONSTRAINT uq_trigger_key UNIQUE (sink_id, scan_id, caller_node_id)
|
||||
);
|
||||
|
||||
-- Indexes for common queries
|
||||
CREATE INDEX idx_vuln_surface_triggers_sink ON scanner.vuln_surface_triggers(sink_id);
|
||||
CREATE INDEX idx_vuln_surface_triggers_scan ON scanner.vuln_surface_triggers(scan_id);
|
||||
CREATE INDEX idx_vuln_surface_triggers_bucket ON scanner.vuln_surface_triggers(reachability_bucket);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surface_triggers_sink ON vuln_surface_triggers(sink_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surface_triggers_scan ON vuln_surface_triggers(scan_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_vuln_surface_triggers_bucket ON vuln_surface_triggers(reachability_bucket);
|
||||
|
||||
COMMENT ON TABLE scanner.vuln_surface_triggers IS 'Links between vulnerability sink methods and their callers in analyzed code';
|
||||
COMMENT ON TABLE vuln_surface_triggers IS 'Links between vulnerability sink methods and their callers in analyzed code';
|
||||
|
||||
-- =============================================================================
|
||||
-- RLS (Row Level Security)
|
||||
-- =============================================================================
|
||||
ALTER TABLE scanner.vuln_surfaces ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE vuln_surfaces ENABLE ROW LEVEL SECURITY;
|
||||
DROP POLICY IF EXISTS vuln_surfaces_tenant_isolation ON vuln_surfaces;
|
||||
CREATE POLICY vuln_surfaces_tenant_isolation ON vuln_surfaces
|
||||
FOR ALL
|
||||
USING (tenant_id = current_tenant_id())
|
||||
WITH CHECK (tenant_id = current_tenant_id());
|
||||
|
||||
-- Tenant isolation policy
|
||||
CREATE POLICY vuln_surfaces_tenant_isolation ON scanner.vuln_surfaces
|
||||
USING (tenant_id = current_setting('app.tenant_id', true)::uuid);
|
||||
|
||||
-- Note: vuln_surface_sinks and triggers inherit isolation through FK to surfaces
|
||||
-- Note: vuln_surface_sinks and triggers inherit isolation through FK to surfaces.
|
||||
|
||||
-- =============================================================================
|
||||
-- FUNCTIONS
|
||||
-- =============================================================================
|
||||
|
||||
-- Get surface statistics for a CVE
|
||||
CREATE OR REPLACE FUNCTION scanner.get_vuln_surface_stats(
|
||||
CREATE OR REPLACE FUNCTION get_vuln_surface_stats(
|
||||
p_tenant_id UUID,
|
||||
p_cve_id TEXT
|
||||
)
|
||||
@@ -164,14 +155,12 @@ BEGIN
|
||||
vs.fixed_version,
|
||||
vs.changed_method_count,
|
||||
COUNT(DISTINCT vst.id)::BIGINT AS trigger_count
|
||||
FROM scanner.vuln_surfaces vs
|
||||
LEFT JOIN scanner.vuln_surface_sinks vss ON vss.surface_id = vs.id
|
||||
LEFT JOIN scanner.vuln_surface_triggers vst ON vst.sink_id = vss.id
|
||||
FROM vuln_surfaces vs
|
||||
LEFT JOIN vuln_surface_sinks vss ON vss.surface_id = vs.id
|
||||
LEFT JOIN vuln_surface_triggers vst ON vst.sink_id = vss.id
|
||||
WHERE vs.tenant_id = p_tenant_id
|
||||
AND vs.cve_id = p_cve_id
|
||||
GROUP BY vs.id, vs.package_ecosystem, vs.package_name, vs.vuln_version, vs.fixed_version, vs.changed_method_count
|
||||
ORDER BY vs.package_ecosystem, vs.package_name;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql STABLE;
|
||||
|
||||
COMMIT;
|
||||
|
||||
@@ -427,10 +427,10 @@ public sealed class PostgresEpssRepository : IEpssRepository
|
||||
FROM {stageTable} s
|
||||
LEFT JOIN {CurrentTable} c ON c.cve_id = s.cve_id
|
||||
CROSS JOIN (
|
||||
SELECT high_score, high_percentile, big_jump_delta
|
||||
FROM {ConfigTable}
|
||||
WHERE org_id IS NULL
|
||||
LIMIT 1
|
||||
SELECT
|
||||
COALESCE((SELECT high_score FROM {ConfigTable} WHERE org_id IS NULL LIMIT 1), 0.50) AS high_score,
|
||||
COALESCE((SELECT high_percentile FROM {ConfigTable} WHERE org_id IS NULL LIMIT 1), 0.95) AS high_percentile,
|
||||
COALESCE((SELECT big_jump_delta FROM {ConfigTable} WHERE org_id IS NULL LIMIT 1), 0.10) AS big_jump_delta
|
||||
) cfg
|
||||
""";
|
||||
|
||||
@@ -493,15 +493,15 @@ public sealed class PostgresEpssRepository : IEpssRepository
|
||||
SELECT
|
||||
cve_id,
|
||||
flags,
|
||||
prev_score,
|
||||
old_score,
|
||||
old_percentile,
|
||||
new_score,
|
||||
new_percentile,
|
||||
prev_band,
|
||||
model_date
|
||||
FROM {ChangesTable}
|
||||
WHERE model_date = @ModelDate
|
||||
{(flags.HasValue ? "AND (flags & @Flags) != 0" : "")}
|
||||
ORDER BY new_score DESC
|
||||
ORDER BY new_score DESC, cve_id
|
||||
LIMIT @Limit
|
||||
""";
|
||||
|
||||
@@ -521,10 +521,10 @@ public sealed class PostgresEpssRepository : IEpssRepository
|
||||
{
|
||||
CveId = r.cve_id,
|
||||
Flags = (Core.Epss.EpssChangeFlags)r.flags,
|
||||
PreviousScore = r.prev_score,
|
||||
PreviousScore = r.old_score,
|
||||
NewScore = r.new_score,
|
||||
NewPercentile = r.new_percentile,
|
||||
PreviousBand = (Core.Epss.EpssPriorityBand)r.prev_band,
|
||||
PreviousBand = ComputeBand(r.old_score, r.old_percentile),
|
||||
ModelDate = r.model_date
|
||||
}).ToList();
|
||||
}
|
||||
@@ -533,13 +533,41 @@ public sealed class PostgresEpssRepository : IEpssRepository
|
||||
{
|
||||
public string cve_id { get; set; } = "";
|
||||
public int flags { get; set; }
|
||||
public double? prev_score { get; set; }
|
||||
public double? old_score { get; set; }
|
||||
public double? old_percentile { get; set; }
|
||||
public double new_score { get; set; }
|
||||
public double new_percentile { get; set; }
|
||||
public int prev_band { get; set; }
|
||||
public DateOnly model_date { get; set; }
|
||||
}
|
||||
|
||||
private static Core.Epss.EpssPriorityBand ComputeBand(double? score, double? percentile)
|
||||
{
|
||||
// Keep logic deterministic and aligned with the sprint band thresholds:
|
||||
// CRITICAL >= 99.5%, HIGH >= 99%, MEDIUM >= 90%, LOW otherwise.
|
||||
// (Score-based elevation is handled at higher layers when needed.)
|
||||
if (score is null || percentile is null)
|
||||
{
|
||||
return Core.Epss.EpssPriorityBand.Unknown;
|
||||
}
|
||||
|
||||
if (percentile.Value >= 0.995)
|
||||
{
|
||||
return Core.Epss.EpssPriorityBand.Critical;
|
||||
}
|
||||
|
||||
if (percentile.Value >= 0.99)
|
||||
{
|
||||
return Core.Epss.EpssPriorityBand.High;
|
||||
}
|
||||
|
||||
if (percentile.Value >= 0.90)
|
||||
{
|
||||
return Core.Epss.EpssPriorityBand.Medium;
|
||||
}
|
||||
|
||||
return Core.Epss.EpssPriorityBand.Low;
|
||||
}
|
||||
|
||||
private sealed class StageCounts
|
||||
{
|
||||
public int distinct_count { get; set; }
|
||||
|
||||
@@ -22,7 +22,7 @@ public sealed class DotNetEntrypointResolverTests
|
||||
|
||||
var entrypoint = entrypoints[0];
|
||||
Assert.Equal("Sample.App", entrypoint.Name);
|
||||
Assert.Equal("Sample.App:Microsoft.AspNetCore.App@10.0.0+Microsoft.NETCore.App@10.0.0+net10.0:any+linux+linux-x64+unix+win+win-x86:frameworkdependent", entrypoint.Id);
|
||||
Assert.Equal("Sample.App:Microsoft.AspNetCore.App@10.0.0+Microsoft.NETCore.App@10.0.0+net10.0:any+linux+linux-x64+unix+win+win-x86:frameworkdependent:no-mvid", entrypoint.Id);
|
||||
Assert.Contains("net10.0", entrypoint.TargetFrameworks);
|
||||
Assert.Contains("linux-x64", entrypoint.RuntimeIdentifiers);
|
||||
Assert.Equal("Sample.App.deps.json", entrypoint.RelativeDepsPath);
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ScaCatalogueDeterminismTests.cs
|
||||
// Sprint: SPRINT_0351_0001_0001_sca_failure_catalogue_completion
|
||||
// Task: SCA-0351-010
|
||||
// Tasks: SCA-0351-010
|
||||
// Description: Determinism validation for SCA Failure Catalogue fixtures
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Core.Tests.Fixtures;
|
||||
|
||||
@@ -18,9 +21,10 @@ namespace StellaOps.Scanner.Core.Tests.Fixtures;
|
||||
/// 2. Reproducible (same content produces same hash)
|
||||
/// 3. Tamper-evident (changes are detectable)
|
||||
/// </summary>
|
||||
public class ScaCatalogueDeterminismTests
|
||||
public sealed class ScaCatalogueDeterminismTests
|
||||
{
|
||||
private const string CatalogueBasePath = "../../../../../../tests/fixtures/sca/catalogue";
|
||||
private static readonly string CatalogueBasePath = Path.GetFullPath(
|
||||
Path.Combine(AppContext.BaseDirectory, "../../../../../../../tests/fixtures/sca/catalogue"));
|
||||
|
||||
[Theory]
|
||||
[InlineData("fc6")]
|
||||
@@ -33,12 +37,11 @@ public class ScaCatalogueDeterminismTests
|
||||
var fixturePath = Path.Combine(CatalogueBasePath, fixtureId);
|
||||
if (!Directory.Exists(fixturePath)) return;
|
||||
|
||||
// Compute hash of all fixture files
|
||||
var hash1 = ComputeFixtureHash(fixturePath);
|
||||
var hash2 = ComputeFixtureHash(fixturePath);
|
||||
|
||||
Assert.Equal(hash1, hash2);
|
||||
Assert.NotEmpty(hash1);
|
||||
Assert.False(string.IsNullOrWhiteSpace(hash1));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -47,19 +50,18 @@ public class ScaCatalogueDeterminismTests
|
||||
[InlineData("fc8")]
|
||||
[InlineData("fc9")]
|
||||
[InlineData("fc10")]
|
||||
public void Fixture_ManifestHasRequiredFields(string fixtureId)
|
||||
public void Fixture_ExpectedJsonHasRequiredFields(string fixtureId)
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, fixtureId, "manifest.json");
|
||||
if (!File.Exists(manifestPath)) return;
|
||||
var expectedPath = Path.Combine(CatalogueBasePath, fixtureId, "expected.json");
|
||||
if (!File.Exists(expectedPath)) return;
|
||||
|
||||
var json = File.ReadAllText(manifestPath);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
using var doc = JsonDocument.Parse(File.ReadAllText(expectedPath));
|
||||
var root = doc.RootElement;
|
||||
|
||||
// Required fields for deterministic fixtures
|
||||
Assert.True(root.TryGetProperty("id", out _), "manifest missing 'id'");
|
||||
Assert.True(root.TryGetProperty("description", out _), "manifest missing 'description'");
|
||||
Assert.True(root.TryGetProperty("failureMode", out _), "manifest missing 'failureMode'");
|
||||
Assert.True(root.TryGetProperty("id", out _), "expected.json missing 'id'");
|
||||
Assert.True(root.TryGetProperty("description", out _), "expected.json missing 'description'");
|
||||
Assert.True(root.TryGetProperty("failure_mode", out _), "expected.json missing 'failure_mode'");
|
||||
Assert.True(root.TryGetProperty("expected_findings", out _), "expected.json missing 'expected_findings'");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -80,20 +82,14 @@ public class ScaCatalogueDeterminismTests
|
||||
var content = File.ReadAllText(file);
|
||||
|
||||
// Check for common external URL patterns that would break offline operation
|
||||
Assert.DoesNotContain("http://", content.ToLowerInvariant().Replace("https://", ""));
|
||||
Assert.DoesNotContain("http://", content.ToLowerInvariant().Replace("https://", string.Empty, StringComparison.Ordinal));
|
||||
|
||||
// Allow https only for documentation references, not actual fetches
|
||||
var httpsCount = CountOccurrences(content.ToLowerInvariant(), "https://");
|
||||
if (httpsCount > 0)
|
||||
{
|
||||
// If HTTPS URLs exist, they should be in comments or documentation
|
||||
// Real fixtures shouldn't require network access
|
||||
var extension = Path.GetExtension(file).ToLowerInvariant();
|
||||
if (extension is ".json" or ".yaml" or ".yml")
|
||||
{
|
||||
// For data files, URLs should only be in documentation fields
|
||||
// This is a soft check - actual network isolation is tested elsewhere
|
||||
}
|
||||
// Soft check only; actual network isolation is tested elsewhere.
|
||||
_ = Path.GetExtension(file).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -109,7 +105,6 @@ public class ScaCatalogueDeterminismTests
|
||||
var fixturePath = Path.Combine(CatalogueBasePath, fixtureId);
|
||||
if (!Directory.Exists(fixturePath)) return;
|
||||
|
||||
// File ordering should be deterministic
|
||||
var files1 = Directory.GetFiles(fixturePath, "*", SearchOption.AllDirectories)
|
||||
.Select(f => Path.GetRelativePath(fixturePath, f))
|
||||
.OrderBy(f => f, StringComparer.Ordinal)
|
||||
@@ -129,7 +124,6 @@ public class ScaCatalogueDeterminismTests
|
||||
var inputsLockPath = Path.Combine(CatalogueBasePath, "inputs.lock");
|
||||
if (!File.Exists(inputsLockPath)) return;
|
||||
|
||||
// Compute hash twice
|
||||
var bytes = File.ReadAllBytes(inputsLockPath);
|
||||
var hash1 = SHA256.HashData(bytes);
|
||||
var hash2 = SHA256.HashData(bytes);
|
||||
@@ -145,7 +139,6 @@ public class ScaCatalogueDeterminismTests
|
||||
|
||||
var content = File.ReadAllText(inputsLockPath);
|
||||
|
||||
// All FC6-FC10 fixtures should be referenced
|
||||
Assert.Contains("fc6", content.ToLowerInvariant());
|
||||
Assert.Contains("fc7", content.ToLowerInvariant());
|
||||
Assert.Contains("fc8", content.ToLowerInvariant());
|
||||
@@ -153,17 +146,13 @@ public class ScaCatalogueDeterminismTests
|
||||
Assert.Contains("fc10", content.ToLowerInvariant());
|
||||
}
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static string ComputeFixtureHash(string fixturePath)
|
||||
{
|
||||
var files = Directory.GetFiles(fixturePath, "*", SearchOption.AllDirectories)
|
||||
.OrderBy(f => f, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
using var sha256 = SHA256.Create();
|
||||
var combined = new StringBuilder();
|
||||
|
||||
foreach (var file in files)
|
||||
{
|
||||
var relativePath = Path.GetRelativePath(fixturePath, file);
|
||||
@@ -185,8 +174,7 @@ public class ScaCatalogueDeterminismTests
|
||||
count++;
|
||||
index += pattern.Length;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -1,213 +1,31 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ScaFailureCatalogueTests.cs
|
||||
// Sprint: SPRINT_0351_0001_0001_sca_failure_catalogue_completion
|
||||
// Task: SCA-0351-008
|
||||
// Description: xUnit tests for SCA Failure Catalogue FC6-FC10
|
||||
// Tasks: SCA-0351-008, SCA-0351-010
|
||||
// Description: Validates FC6-FC10 fixture presence, structure, and DSSE binding.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Core.Tests.Fixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for SCA Failure Catalogue cases FC6-FC10.
|
||||
/// Each test validates that the scanner correctly handles a specific real-world failure mode.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Fixture directory: tests/fixtures/sca/catalogue/
|
||||
///
|
||||
/// FC6: Java Shadow JAR - Fat/uber JARs with shaded dependencies
|
||||
/// FC7: .NET Transitive Pinning - Transitive dependency version conflicts
|
||||
/// FC8: Docker Multi-Stage Leakage - Build-time dependencies in runtime
|
||||
/// FC9: PURL Namespace Collision - Same package name in different ecosystems
|
||||
/// FC10: CVE Split/Merge - Vulnerability split across multiple CVEs
|
||||
/// </remarks>
|
||||
public class ScaFailureCatalogueTests
|
||||
public sealed class ScaFailureCatalogueTests
|
||||
{
|
||||
private const string CatalogueBasePath = "../../../../../../tests/fixtures/sca/catalogue";
|
||||
|
||||
#region FC6: Java Shadow JAR
|
||||
|
||||
[Fact]
|
||||
public void FC6_ShadowJar_ManifestExists()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc6", "manifest.json");
|
||||
Assert.True(File.Exists(manifestPath), $"FC6 manifest not found at {manifestPath}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC6_ShadowJar_HasExpectedFiles()
|
||||
{
|
||||
var fc6Path = Path.Combine(CatalogueBasePath, "fc6");
|
||||
Assert.True(Directory.Exists(fc6Path), "FC6 directory not found");
|
||||
|
||||
var files = Directory.GetFiles(fc6Path, "*", SearchOption.AllDirectories);
|
||||
Assert.NotEmpty(files);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC6_ShadowJar_ManifestIsValid()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc6", "manifest.json");
|
||||
if (!File.Exists(manifestPath)) return; // Skip if not present
|
||||
|
||||
var json = File.ReadAllText(manifestPath);
|
||||
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
|
||||
|
||||
Assert.NotNull(manifest);
|
||||
Assert.Equal("FC6", manifest.Id);
|
||||
Assert.NotEmpty(manifest.Description);
|
||||
Assert.NotEmpty(manifest.ExpectedFindings);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region FC7: .NET Transitive Pinning
|
||||
|
||||
[Fact]
|
||||
public void FC7_TransitivePinning_ManifestExists()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc7", "manifest.json");
|
||||
Assert.True(File.Exists(manifestPath), $"FC7 manifest not found at {manifestPath}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC7_TransitivePinning_HasExpectedFiles()
|
||||
{
|
||||
var fc7Path = Path.Combine(CatalogueBasePath, "fc7");
|
||||
Assert.True(Directory.Exists(fc7Path), "FC7 directory not found");
|
||||
|
||||
var files = Directory.GetFiles(fc7Path, "*", SearchOption.AllDirectories);
|
||||
Assert.NotEmpty(files);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC7_TransitivePinning_ManifestIsValid()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc7", "manifest.json");
|
||||
if (!File.Exists(manifestPath)) return;
|
||||
|
||||
var json = File.ReadAllText(manifestPath);
|
||||
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
|
||||
|
||||
Assert.NotNull(manifest);
|
||||
Assert.Equal("FC7", manifest.Id);
|
||||
Assert.NotEmpty(manifest.ExpectedFindings);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region FC8: Docker Multi-Stage Leakage
|
||||
|
||||
[Fact]
|
||||
public void FC8_MultiStageLeakage_ManifestExists()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc8", "manifest.json");
|
||||
Assert.True(File.Exists(manifestPath), $"FC8 manifest not found at {manifestPath}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC8_MultiStageLeakage_HasDockerfile()
|
||||
{
|
||||
var fc8Path = Path.Combine(CatalogueBasePath, "fc8");
|
||||
Assert.True(Directory.Exists(fc8Path), "FC8 directory not found");
|
||||
|
||||
// Multi-stage leakage tests should have Dockerfile examples
|
||||
var dockerfiles = Directory.GetFiles(fc8Path, "Dockerfile*", SearchOption.AllDirectories);
|
||||
Assert.NotEmpty(dockerfiles);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC8_MultiStageLeakage_ManifestIsValid()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc8", "manifest.json");
|
||||
if (!File.Exists(manifestPath)) return;
|
||||
|
||||
var json = File.ReadAllText(manifestPath);
|
||||
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
|
||||
|
||||
Assert.NotNull(manifest);
|
||||
Assert.Equal("FC8", manifest.Id);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region FC9: PURL Namespace Collision
|
||||
|
||||
[Fact]
|
||||
public void FC9_PurlNamespaceCollision_ManifestExists()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc9", "manifest.json");
|
||||
Assert.True(File.Exists(manifestPath), $"FC9 manifest not found at {manifestPath}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC9_PurlNamespaceCollision_HasMultipleEcosystems()
|
||||
{
|
||||
var fc9Path = Path.Combine(CatalogueBasePath, "fc9");
|
||||
Assert.True(Directory.Exists(fc9Path), "FC9 directory not found");
|
||||
|
||||
// Should contain files for multiple ecosystems
|
||||
var files = Directory.GetFiles(fc9Path, "*", SearchOption.AllDirectories)
|
||||
.Select(f => Path.GetFileName(f))
|
||||
.ToList();
|
||||
|
||||
Assert.NotEmpty(files);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC9_PurlNamespaceCollision_ManifestIsValid()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc9", "manifest.json");
|
||||
if (!File.Exists(manifestPath)) return;
|
||||
|
||||
var json = File.ReadAllText(manifestPath);
|
||||
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
|
||||
|
||||
Assert.NotNull(manifest);
|
||||
Assert.Equal("FC9", manifest.Id);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region FC10: CVE Split/Merge
|
||||
|
||||
[Fact]
|
||||
public void FC10_CveSplitMerge_ManifestExists()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc10", "manifest.json");
|
||||
Assert.True(File.Exists(manifestPath), $"FC10 manifest not found at {manifestPath}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC10_CveSplitMerge_ManifestIsValid()
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, "fc10", "manifest.json");
|
||||
if (!File.Exists(manifestPath)) return;
|
||||
|
||||
var json = File.ReadAllText(manifestPath);
|
||||
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
|
||||
|
||||
Assert.NotNull(manifest);
|
||||
Assert.Equal("FC10", manifest.Id);
|
||||
|
||||
// CVE split/merge should have multiple related CVEs
|
||||
Assert.NotNull(manifest.RelatedCves);
|
||||
Assert.True(manifest.RelatedCves.Count >= 2, "CVE split/merge should have at least 2 related CVEs");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Cross-Catalogue Tests
|
||||
private static readonly string CatalogueBasePath = Path.GetFullPath(
|
||||
Path.Combine(AppContext.BaseDirectory, "../../../../../../../tests/fixtures/sca/catalogue"));
|
||||
|
||||
[Fact]
|
||||
public void AllCatalogueFixtures_HaveInputsLock()
|
||||
{
|
||||
var inputsLockPath = Path.Combine(CatalogueBasePath, "inputs.lock");
|
||||
Assert.True(File.Exists(inputsLockPath), "inputs.lock not found");
|
||||
Assert.True(File.Exists(inputsLockPath), $"inputs.lock not found at {inputsLockPath}");
|
||||
|
||||
var content = File.ReadAllText(inputsLockPath);
|
||||
Assert.NotEmpty(content);
|
||||
Assert.False(string.IsNullOrWhiteSpace(content));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -218,8 +36,8 @@ public class ScaFailureCatalogueTests
|
||||
[InlineData("fc10")]
|
||||
public void CatalogueFixture_DirectoryExists(string fixtureId)
|
||||
{
|
||||
var fixturePath = Path.Combine(CatalogueBasePath, fixtureId);
|
||||
Assert.True(Directory.Exists(fixturePath), $"Fixture {fixtureId} directory not found");
|
||||
var fixturePath = FixturePath(fixtureId);
|
||||
Assert.True(Directory.Exists(fixturePath), $"Fixture {fixtureId} directory not found at {fixturePath}");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
@@ -228,68 +46,157 @@ public class ScaFailureCatalogueTests
|
||||
[InlineData("fc8")]
|
||||
[InlineData("fc9")]
|
||||
[InlineData("fc10")]
|
||||
public void CatalogueFixture_HasManifest(string fixtureId)
|
||||
public void CatalogueFixture_HasExpectedJson(string fixtureId)
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, fixtureId, "manifest.json");
|
||||
Assert.True(File.Exists(manifestPath), $"Fixture {fixtureId} manifest not found");
|
||||
var expectedPath = ExpectedJsonPath(fixtureId);
|
||||
Assert.True(File.Exists(expectedPath), $"Fixture {fixtureId} expected.json not found at {expectedPath}");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Determinism Tests
|
||||
|
||||
[Theory]
|
||||
[InlineData("fc6")]
|
||||
[InlineData("fc7")]
|
||||
[InlineData("fc8")]
|
||||
[InlineData("fc9")]
|
||||
[InlineData("fc10")]
|
||||
public void CatalogueFixture_ManifestIsDeterministic(string fixtureId)
|
||||
public void CatalogueFixture_HasInputTxt(string fixtureId)
|
||||
{
|
||||
var manifestPath = Path.Combine(CatalogueBasePath, fixtureId, "manifest.json");
|
||||
if (!File.Exists(manifestPath)) return;
|
||||
var inputPath = InputTxtPath(fixtureId);
|
||||
Assert.True(File.Exists(inputPath), $"Fixture {fixtureId} input.txt not found at {inputPath}");
|
||||
|
||||
// Read twice and ensure identical
|
||||
var content1 = File.ReadAllText(manifestPath);
|
||||
var content2 = File.ReadAllText(manifestPath);
|
||||
Assert.Equal(content1, content2);
|
||||
|
||||
// Verify can be parsed to consistent structure
|
||||
var manifest1 = JsonSerializer.Deserialize<CatalogueManifest>(content1);
|
||||
var manifest2 = JsonSerializer.Deserialize<CatalogueManifest>(content2);
|
||||
|
||||
Assert.NotNull(manifest1);
|
||||
Assert.NotNull(manifest2);
|
||||
Assert.Equal(manifest1.Id, manifest2.Id);
|
||||
Assert.Equal(manifest1.Description, manifest2.Description);
|
||||
var content = File.ReadAllText(inputPath);
|
||||
Assert.False(string.IsNullOrWhiteSpace(content));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Test Models
|
||||
|
||||
private record CatalogueManifest
|
||||
[Theory]
|
||||
[InlineData("fc6")]
|
||||
[InlineData("fc7")]
|
||||
[InlineData("fc8")]
|
||||
[InlineData("fc9")]
|
||||
[InlineData("fc10")]
|
||||
public void CatalogueFixture_HasDsseManifest(string fixtureId)
|
||||
{
|
||||
public string Id { get; init; } = "";
|
||||
public string Description { get; init; } = "";
|
||||
public string FailureMode { get; init; } = "";
|
||||
public List<ExpectedFinding> ExpectedFindings { get; init; } = [];
|
||||
public List<string> RelatedCves { get; init; } = [];
|
||||
public DsseManifest? Dsse { get; init; }
|
||||
var dssePath = DsseManifestPath(fixtureId);
|
||||
Assert.True(File.Exists(dssePath), $"Fixture {fixtureId} manifest.dsse.json not found at {dssePath}");
|
||||
}
|
||||
|
||||
private record ExpectedFinding
|
||||
[Theory]
|
||||
[InlineData("fc6")]
|
||||
[InlineData("fc7")]
|
||||
[InlineData("fc8")]
|
||||
[InlineData("fc9")]
|
||||
[InlineData("fc10")]
|
||||
public void CatalogueFixture_DssePayloadMatchesExpectedJson(string fixtureId)
|
||||
{
|
||||
public string Purl { get; init; } = "";
|
||||
public string VulnerabilityId { get; init; } = "";
|
||||
public string ExpectedResult { get; init; } = "";
|
||||
var expectedPath = ExpectedJsonPath(fixtureId);
|
||||
var dssePath = DsseManifestPath(fixtureId);
|
||||
|
||||
if (!File.Exists(expectedPath) || !File.Exists(dssePath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var expected = NormalizeLineEndings(File.ReadAllText(expectedPath)).TrimEnd();
|
||||
var payload = NormalizeLineEndings(ReadDssePayload(dssePath)).TrimEnd();
|
||||
Assert.Equal(expected, payload);
|
||||
|
||||
using var expectedDoc = JsonDocument.Parse(expected);
|
||||
using var payloadDoc = JsonDocument.Parse(payload);
|
||||
|
||||
Assert.Equal(
|
||||
expectedDoc.RootElement.GetProperty("id").GetString(),
|
||||
payloadDoc.RootElement.GetProperty("id").GetString());
|
||||
}
|
||||
|
||||
private record DsseManifest
|
||||
[Theory]
|
||||
[InlineData("fc6")]
|
||||
[InlineData("fc7")]
|
||||
[InlineData("fc8")]
|
||||
[InlineData("fc9")]
|
||||
[InlineData("fc10")]
|
||||
public void CatalogueFixture_ExpectedJsonHasRequiredFields(string fixtureId)
|
||||
{
|
||||
public string PayloadType { get; init; } = "";
|
||||
public string Signature { get; init; } = "";
|
||||
var expectedPath = ExpectedJsonPath(fixtureId);
|
||||
if (!File.Exists(expectedPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(File.ReadAllText(expectedPath));
|
||||
var root = document.RootElement;
|
||||
|
||||
Assert.True(root.TryGetProperty("id", out var idNode));
|
||||
Assert.False(string.IsNullOrWhiteSpace(idNode.GetString()));
|
||||
|
||||
Assert.True(root.TryGetProperty("description", out var descriptionNode));
|
||||
Assert.False(string.IsNullOrWhiteSpace(descriptionNode.GetString()));
|
||||
|
||||
Assert.True(root.TryGetProperty("failure_mode", out var failureModeNode));
|
||||
Assert.Equal(JsonValueKind.Object, failureModeNode.ValueKind);
|
||||
|
||||
Assert.True(root.TryGetProperty("expected_findings", out var findingsNode));
|
||||
Assert.Equal(JsonValueKind.Array, findingsNode.ValueKind);
|
||||
Assert.True(findingsNode.GetArrayLength() > 0);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
[Fact]
|
||||
public void FC8_MultiStageLeakage_HasDockerfileFixture()
|
||||
{
|
||||
var dockerfilePath = Path.Combine(FixturePath("fc8"), "Dockerfile.multistage");
|
||||
Assert.True(File.Exists(dockerfilePath), $"FC8 Dockerfile fixture not found at {dockerfilePath}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC9_PurlNamespaceCollision_HasMultipleEcosystems()
|
||||
{
|
||||
using var document = JsonDocument.Parse(File.ReadAllText(ExpectedJsonPath("fc9")));
|
||||
var root = document.RootElement;
|
||||
|
||||
var ecosystems = root
|
||||
.GetProperty("input")
|
||||
.GetProperty("ecosystems");
|
||||
|
||||
Assert.Equal(JsonValueKind.Array, ecosystems.ValueKind);
|
||||
Assert.True(ecosystems.GetArrayLength() >= 2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FC10_CveSplitMerge_HasMultipleRelatedCves()
|
||||
{
|
||||
using var document = JsonDocument.Parse(File.ReadAllText(ExpectedJsonPath("fc10")));
|
||||
var root = document.RootElement;
|
||||
|
||||
var cveCases = root.GetProperty("cve_cases");
|
||||
|
||||
var splitCves = cveCases.GetProperty("split").GetProperty("split_cves");
|
||||
var mergedCves = cveCases.GetProperty("merge").GetProperty("merged_cves");
|
||||
var chainCves = cveCases.GetProperty("chain").GetProperty("cve_chain");
|
||||
|
||||
var total = splitCves.GetArrayLength() + mergedCves.GetArrayLength() + chainCves.GetArrayLength();
|
||||
Assert.True(total >= 2, "FC10 should capture at least two related CVEs across split/merge/chain cases.");
|
||||
}
|
||||
|
||||
private static string FixturePath(string fixtureId)
|
||||
=> Path.Combine(CatalogueBasePath, fixtureId);
|
||||
|
||||
private static string ExpectedJsonPath(string fixtureId)
|
||||
=> Path.Combine(FixturePath(fixtureId), "expected.json");
|
||||
|
||||
private static string DsseManifestPath(string fixtureId)
|
||||
=> Path.Combine(FixturePath(fixtureId), "manifest.dsse.json");
|
||||
|
||||
private static string InputTxtPath(string fixtureId)
|
||||
=> Path.Combine(FixturePath(fixtureId), "input.txt");
|
||||
|
||||
private static string NormalizeLineEndings(string value)
|
||||
=> value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace("\r", "\n", StringComparison.Ordinal);
|
||||
|
||||
private static string ReadDssePayload(string dsseManifestPath)
|
||||
{
|
||||
using var envelope = JsonDocument.Parse(File.ReadAllText(dsseManifestPath));
|
||||
var payloadB64 = envelope.RootElement.GetProperty("payload").GetString();
|
||||
Assert.False(string.IsNullOrWhiteSpace(payloadB64), $"DSSE payload missing in {dsseManifestPath}");
|
||||
|
||||
var payloadBytes = Convert.FromBase64String(payloadB64!);
|
||||
return Encoding.UTF8.GetString(payloadBytes);
|
||||
}
|
||||
}
|
||||
@@ -28,10 +28,13 @@ public sealed class LayeredRootFileSystemTests : IDisposable
|
||||
var entrypointPath = Path.Combine(usrBin1, "entrypoint.sh");
|
||||
File.WriteAllText(entrypointPath, "#!/bin/sh\necho layer1\n");
|
||||
#if NET8_0_OR_GREATER
|
||||
File.SetUnixFileMode(entrypointPath,
|
||||
UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
|
||||
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
|
||||
UnixFileMode.OtherRead | UnixFileMode.OtherExecute);
|
||||
if (!OperatingSystem.IsWindows())
|
||||
{
|
||||
File.SetUnixFileMode(entrypointPath,
|
||||
UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
|
||||
UnixFileMode.GroupRead | UnixFileMode.GroupExecute |
|
||||
UnixFileMode.OtherRead | UnixFileMode.OtherExecute);
|
||||
}
|
||||
#endif
|
||||
|
||||
var optDirectory1 = Path.Combine(layer1, "opt");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using StellaOps.Scanner.Core.Epss;
|
||||
using StellaOps.Scanner.Storage.Epss;
|
||||
using Xunit;
|
||||
|
||||
@@ -39,4 +40,3 @@ public sealed class EpssChangeDetectorTests
|
||||
Assert.Equal(EpssChangeFlags.NewScored | EpssChangeFlags.TopPercentile, newScored);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Core.Epss;
|
||||
using StellaOps.Scanner.Storage.Epss;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Tests;
|
||||
|
||||
[Collection("scanner-postgres")]
|
||||
public sealed class EpssRepositoryChangesIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private readonly ScannerPostgresFixture _fixture;
|
||||
private ScannerDataSource _dataSource = null!;
|
||||
private PostgresEpssRepository _repository = null!;
|
||||
|
||||
public EpssRepositoryChangesIntegrationTests(ScannerPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.TruncateAllTablesAsync();
|
||||
|
||||
var options = new ScannerStorageOptions
|
||||
{
|
||||
Postgres = new StellaOps.Infrastructure.Postgres.Options.PostgresOptions
|
||||
{
|
||||
ConnectionString = _fixture.ConnectionString,
|
||||
SchemaName = _fixture.SchemaName
|
||||
}
|
||||
};
|
||||
|
||||
_dataSource = new ScannerDataSource(Options.Create(options), NullLogger<ScannerDataSource>.Instance);
|
||||
_repository = new PostgresEpssRepository(_dataSource);
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task GetChangesAsync_ReturnsMappedFieldsAndSupportsFlagFiltering()
|
||||
{
|
||||
var thresholds = EpssChangeDetector.DefaultThresholds;
|
||||
|
||||
var day1 = new DateOnly(2027, 1, 15);
|
||||
var run1 = await _repository.BeginImportAsync(day1, "bundle://day1.csv.gz", DateTimeOffset.Parse("2027-01-15T00:05:00Z"), "sha256:day1");
|
||||
|
||||
var day1Rows = new[]
|
||||
{
|
||||
new EpssScoreRow("CVE-2024-0001", 0.40, 0.90),
|
||||
new EpssScoreRow("CVE-2024-0002", 0.60, 0.96)
|
||||
};
|
||||
|
||||
var write1 = await _repository.WriteSnapshotAsync(run1.ImportRunId, day1, DateTimeOffset.Parse("2027-01-15T00:06:00Z"), ToAsync(day1Rows));
|
||||
await _repository.MarkImportSucceededAsync(run1.ImportRunId, write1.RowCount, decompressedSha256: "sha256:decompressed1", modelVersionTag: "v2027.01.15", publishedDate: day1);
|
||||
|
||||
var day2 = new DateOnly(2027, 1, 16);
|
||||
var run2 = await _repository.BeginImportAsync(day2, "bundle://day2.csv.gz", DateTimeOffset.Parse("2027-01-16T00:05:00Z"), "sha256:day2");
|
||||
|
||||
var day2Rows = new[]
|
||||
{
|
||||
new EpssScoreRow("CVE-2024-0001", 0.55, 0.95),
|
||||
new EpssScoreRow("CVE-2024-0002", 0.45, 0.94),
|
||||
new EpssScoreRow("CVE-2024-0003", 0.70, 0.97)
|
||||
};
|
||||
|
||||
var write2 = await _repository.WriteSnapshotAsync(run2.ImportRunId, day2, DateTimeOffset.Parse("2027-01-16T00:06:00Z"), ToAsync(day2Rows));
|
||||
await _repository.MarkImportSucceededAsync(run2.ImportRunId, write2.RowCount, decompressedSha256: "sha256:decompressed2", modelVersionTag: "v2027.01.16", publishedDate: day2);
|
||||
|
||||
var changes = await _repository.GetChangesAsync(day2);
|
||||
Assert.Equal(3, changes.Count);
|
||||
|
||||
var byCve = changes.ToDictionary(c => c.CveId, StringComparer.Ordinal);
|
||||
|
||||
Assert.Equal(day2, byCve["CVE-2024-0001"].ModelDate);
|
||||
Assert.Equal(0.40, byCve["CVE-2024-0001"].PreviousScore);
|
||||
Assert.Equal(0.55, byCve["CVE-2024-0001"].NewScore);
|
||||
Assert.Equal(0.95, byCve["CVE-2024-0001"].NewPercentile);
|
||||
Assert.Equal(EpssPriorityBand.Medium, byCve["CVE-2024-0001"].PreviousBand);
|
||||
Assert.Equal(
|
||||
EpssChangeDetector.ComputeFlags(0.40, 0.55, 0.90, 0.95, thresholds),
|
||||
byCve["CVE-2024-0001"].Flags);
|
||||
|
||||
Assert.Equal(0.60, byCve["CVE-2024-0002"].PreviousScore);
|
||||
Assert.Equal(0.45, byCve["CVE-2024-0002"].NewScore);
|
||||
Assert.Equal(0.94, byCve["CVE-2024-0002"].NewPercentile);
|
||||
Assert.Equal(EpssPriorityBand.Medium, byCve["CVE-2024-0002"].PreviousBand);
|
||||
Assert.Equal(
|
||||
EpssChangeDetector.ComputeFlags(0.60, 0.45, 0.96, 0.94, thresholds),
|
||||
byCve["CVE-2024-0002"].Flags);
|
||||
|
||||
Assert.Null(byCve["CVE-2024-0003"].PreviousScore);
|
||||
Assert.Equal(0.70, byCve["CVE-2024-0003"].NewScore);
|
||||
Assert.Equal(0.97, byCve["CVE-2024-0003"].NewPercentile);
|
||||
Assert.Equal(EpssPriorityBand.Unknown, byCve["CVE-2024-0003"].PreviousBand);
|
||||
Assert.Equal(
|
||||
EpssChangeDetector.ComputeFlags(null, 0.70, null, 0.97, thresholds),
|
||||
byCve["CVE-2024-0003"].Flags);
|
||||
|
||||
var crossedHigh = await _repository.GetChangesAsync(day2, EpssChangeFlags.CrossedHigh);
|
||||
Assert.Single(crossedHigh);
|
||||
Assert.Equal("CVE-2024-0001", crossedHigh[0].CveId);
|
||||
|
||||
var newScored = await _repository.GetChangesAsync(day2, EpssChangeFlags.NewScored);
|
||||
Assert.Single(newScored);
|
||||
Assert.Equal("CVE-2024-0003", newScored[0].CveId);
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<EpssScoreRow> ToAsync(IEnumerable<EpssScoreRow> rows)
|
||||
{
|
||||
foreach (var row in rows)
|
||||
{
|
||||
yield return row;
|
||||
await Task.Yield();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
// =============================================================================
|
||||
// EpssEndpointsTests.cs
|
||||
// Sprint: SPRINT_3410_0002_0001_epss_scanner_integration
|
||||
// Task: EPSS-SCAN-011 - Integration tests for EPSS endpoints
|
||||
// =============================================================================
|
||||
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using StellaOps.Scanner.Core.Epss;
|
||||
using StellaOps.Scanner.WebService.Endpoints;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.WebService.Tests;
|
||||
|
||||
[Trait("Category", "Integration")]
|
||||
[Trait("Sprint", "3410.0002")]
|
||||
public sealed class EpssEndpointsTests : IDisposable
|
||||
{
|
||||
private readonly TestSurfaceSecretsScope _secrets;
|
||||
private readonly InMemoryEpssProvider _epssProvider;
|
||||
private readonly ScannerApplicationFactory _factory;
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public EpssEndpointsTests()
|
||||
{
|
||||
_secrets = new TestSurfaceSecretsScope();
|
||||
_epssProvider = new InMemoryEpssProvider();
|
||||
|
||||
_factory = new ScannerApplicationFactory().WithOverrides(
|
||||
configureConfiguration: config => config["scanner:authority:enabled"] = "false",
|
||||
configureServices: services =>
|
||||
{
|
||||
services.RemoveAll<IEpssProvider>();
|
||||
services.AddSingleton<IEpssProvider>(_epssProvider);
|
||||
});
|
||||
|
||||
_client = _factory.CreateClient();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_client.Dispose();
|
||||
_factory.Dispose();
|
||||
_secrets.Dispose();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /epss/current rejects empty CVE list")]
|
||||
public async Task PostCurrentBatch_EmptyList_ReturnsBadRequest()
|
||||
{
|
||||
var response = await _client.PostAsJsonAsync("/api/v1/epss/current", new { cveIds = Array.Empty<string>() });
|
||||
|
||||
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
|
||||
|
||||
var problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.NotNull(problem);
|
||||
Assert.Equal("Invalid request", problem!.Title);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /epss/current rejects >1000 CVEs")]
|
||||
public async Task PostCurrentBatch_OverLimit_ReturnsBadRequest()
|
||||
{
|
||||
var cveIds = Enumerable.Range(1, 1001).Select(i => $"CVE-2025-{i:D5}").ToArray();
|
||||
|
||||
var response = await _client.PostAsJsonAsync("/api/v1/epss/current", new { cveIds });
|
||||
|
||||
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
|
||||
|
||||
var problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.NotNull(problem);
|
||||
Assert.Equal("Batch size exceeded", problem!.Title);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /epss/current returns 503 when EPSS unavailable")]
|
||||
public async Task PostCurrentBatch_WhenUnavailable_Returns503()
|
||||
{
|
||||
_epssProvider.Available = false;
|
||||
|
||||
var response = await _client.PostAsJsonAsync("/api/v1/epss/current", new { cveIds = new[] { "CVE-2021-44228" } });
|
||||
|
||||
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
|
||||
|
||||
var problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.NotNull(problem);
|
||||
Assert.Equal(503, problem!.Status);
|
||||
Assert.Contains("EPSS data is not available", problem.Detail, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "POST /epss/current returns found + notFound results")]
|
||||
public async Task PostCurrentBatch_ReturnsBatchResponse()
|
||||
{
|
||||
_epssProvider.LatestModelDate = new DateOnly(2025, 12, 17);
|
||||
_epssProvider.SetCurrent(EpssEvidence.CreateWithTimestamp(
|
||||
cveId: "CVE-2021-44228",
|
||||
score: 0.97,
|
||||
percentile: 0.99,
|
||||
modelDate: _epssProvider.LatestModelDate.Value,
|
||||
capturedAt: new DateTimeOffset(2025, 12, 18, 12, 0, 0, TimeSpan.Zero),
|
||||
source: "test",
|
||||
fromCache: false));
|
||||
|
||||
_epssProvider.SetCurrent(EpssEvidence.CreateWithTimestamp(
|
||||
cveId: "CVE-2022-22965",
|
||||
score: 0.95,
|
||||
percentile: 0.98,
|
||||
modelDate: _epssProvider.LatestModelDate.Value,
|
||||
capturedAt: new DateTimeOffset(2025, 12, 18, 12, 0, 0, TimeSpan.Zero),
|
||||
source: "test",
|
||||
fromCache: false));
|
||||
|
||||
var response = await _client.PostAsJsonAsync("/api/v1/epss/current", new
|
||||
{
|
||||
cveIds = new[] { "CVE-2021-44228", "CVE-2022-22965", "CVE-1999-0001" }
|
||||
});
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
|
||||
var batch = await response.Content.ReadFromJsonAsync<EpssBatchResponse>();
|
||||
Assert.NotNull(batch);
|
||||
Assert.Equal("2025-12-17", batch!.ModelDate);
|
||||
Assert.Equal(2, batch.Found.Count);
|
||||
Assert.Single(batch.NotFound);
|
||||
Assert.Contains("CVE-1999-0001", batch.NotFound);
|
||||
Assert.Contains(batch.Found, e => e.CveId == "CVE-2021-44228" && Math.Abs(e.Score - 0.97) < 0.0001);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "GET /epss/current/{cveId} returns 404 when not found")]
|
||||
public async Task GetCurrentSingle_NotFound_Returns404()
|
||||
{
|
||||
var response = await _client.GetAsync("/api/v1/epss/current/CVE-1999-0001");
|
||||
|
||||
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
|
||||
|
||||
var problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.NotNull(problem);
|
||||
Assert.Equal("CVE not found", problem!.Title);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "GET /epss/current/{cveId} returns evidence when found")]
|
||||
public async Task GetCurrentSingle_Found_ReturnsEvidence()
|
||||
{
|
||||
_epssProvider.LatestModelDate = new DateOnly(2025, 12, 17);
|
||||
_epssProvider.SetCurrent(EpssEvidence.CreateWithTimestamp(
|
||||
cveId: "CVE-2021-44228",
|
||||
score: 0.97,
|
||||
percentile: 0.99,
|
||||
modelDate: _epssProvider.LatestModelDate.Value,
|
||||
capturedAt: new DateTimeOffset(2025, 12, 18, 12, 0, 0, TimeSpan.Zero),
|
||||
source: "test"));
|
||||
|
||||
var response = await _client.GetAsync("/api/v1/epss/current/CVE-2021-44228");
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
|
||||
var evidence = await response.Content.ReadFromJsonAsync<EpssEvidence>();
|
||||
Assert.NotNull(evidence);
|
||||
Assert.Equal("CVE-2021-44228", evidence!.CveId);
|
||||
Assert.Equal(0.97, evidence.Score, 5);
|
||||
Assert.Equal(new DateOnly(2025, 12, 17), evidence.ModelDate);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "GET /epss/history/{cveId} rejects invalid date formats")]
|
||||
public async Task GetHistory_InvalidDates_ReturnsBadRequest()
|
||||
{
|
||||
var response = await _client.GetAsync("/api/v1/epss/history/CVE-2021-44228?startDate=2025-99-99&endDate=2025-12-17");
|
||||
|
||||
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
|
||||
|
||||
var problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.NotNull(problem);
|
||||
Assert.Equal("Invalid date format", problem!.Title);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "GET /epss/history/{cveId} returns 404 when no history exists")]
|
||||
public async Task GetHistory_NoHistory_Returns404()
|
||||
{
|
||||
var response = await _client.GetAsync("/api/v1/epss/history/CVE-2021-44228?startDate=2025-12-15&endDate=2025-12-17");
|
||||
|
||||
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
|
||||
|
||||
var problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
|
||||
Assert.NotNull(problem);
|
||||
Assert.Equal("No history found", problem!.Title);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "GET /epss/history/{cveId} returns history for date range")]
|
||||
public async Task GetHistory_ReturnsHistoryResponse()
|
||||
{
|
||||
var cveId = "CVE-2021-44228";
|
||||
var capturedAt = new DateTimeOffset(2025, 12, 18, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
_epssProvider.SetHistory(
|
||||
cveId,
|
||||
new[]
|
||||
{
|
||||
EpssEvidence.CreateWithTimestamp(cveId, 0.10, 0.20, new DateOnly(2025, 12, 15), capturedAt, source: "test"),
|
||||
EpssEvidence.CreateWithTimestamp(cveId, 0.11, 0.21, new DateOnly(2025, 12, 16), capturedAt, source: "test"),
|
||||
EpssEvidence.CreateWithTimestamp(cveId, 0.12, 0.22, new DateOnly(2025, 12, 17), capturedAt, source: "test"),
|
||||
});
|
||||
|
||||
var response = await _client.GetAsync($"/api/v1/epss/history/{cveId}?startDate=2025-12-15&endDate=2025-12-17");
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
|
||||
var history = await response.Content.ReadFromJsonAsync<EpssHistoryResponse>();
|
||||
Assert.NotNull(history);
|
||||
Assert.Equal(cveId, history!.CveId);
|
||||
Assert.Equal("2025-12-15", history.StartDate);
|
||||
Assert.Equal("2025-12-17", history.EndDate);
|
||||
Assert.Equal(3, history.History.Count);
|
||||
Assert.Equal(new DateOnly(2025, 12, 15), history.History[0].ModelDate);
|
||||
Assert.Equal(new DateOnly(2025, 12, 17), history.History[^1].ModelDate);
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "GET /epss/status returns provider availability + model date")]
|
||||
public async Task GetStatus_ReturnsStatus()
|
||||
{
|
||||
_epssProvider.Available = true;
|
||||
_epssProvider.LatestModelDate = new DateOnly(2025, 12, 17);
|
||||
|
||||
var response = await _client.GetAsync("/api/v1/epss/status");
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
|
||||
var status = await response.Content.ReadFromJsonAsync<EpssStatusResponse>();
|
||||
Assert.NotNull(status);
|
||||
Assert.True(status!.Available);
|
||||
Assert.Equal("2025-12-17", status.LatestModelDate);
|
||||
Assert.NotEqual(default, status.LastCheckedUtc);
|
||||
}
|
||||
|
||||
private sealed class InMemoryEpssProvider : IEpssProvider
|
||||
{
|
||||
private readonly Dictionary<string, EpssEvidence> _current = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, List<EpssEvidence>> _history = new(StringComparer.Ordinal);
|
||||
|
||||
public bool Available { get; set; } = true;
|
||||
|
||||
public DateOnly? LatestModelDate { get; set; }
|
||||
|
||||
public Task<EpssEvidence?> GetCurrentAsync(string cveId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(cveId))
|
||||
{
|
||||
return Task.FromResult<EpssEvidence?>(null);
|
||||
}
|
||||
|
||||
var key = NormalizeCveId(cveId);
|
||||
return Task.FromResult(_current.TryGetValue(key, out var evidence) ? evidence : null);
|
||||
}
|
||||
|
||||
public Task<EpssBatchResult> GetCurrentBatchAsync(IEnumerable<string> cveIds, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var found = new List<EpssEvidence>();
|
||||
var notFound = new List<string>();
|
||||
|
||||
foreach (var raw in cveIds ?? Array.Empty<string>())
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(raw))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var key = NormalizeCveId(raw);
|
||||
if (_current.TryGetValue(key, out var evidence))
|
||||
{
|
||||
found.Add(evidence);
|
||||
}
|
||||
else
|
||||
{
|
||||
notFound.Add(raw);
|
||||
}
|
||||
}
|
||||
|
||||
var modelDate = LatestModelDate
|
||||
?? found.Select(static e => e.ModelDate).FirstOrDefault();
|
||||
|
||||
return Task.FromResult(new EpssBatchResult
|
||||
{
|
||||
Found = found,
|
||||
NotFound = notFound,
|
||||
ModelDate = modelDate == default ? new DateOnly(1970, 1, 1) : modelDate,
|
||||
LookupTimeMs = 0,
|
||||
PartiallyFromCache = false
|
||||
});
|
||||
}
|
||||
|
||||
public Task<EpssEvidence?> GetAsOfDateAsync(string cveId, DateOnly asOfDate, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(cveId))
|
||||
{
|
||||
return Task.FromResult<EpssEvidence?>(null);
|
||||
}
|
||||
|
||||
var key = NormalizeCveId(cveId);
|
||||
if (!_history.TryGetValue(key, out var list))
|
||||
{
|
||||
return Task.FromResult<EpssEvidence?>(null);
|
||||
}
|
||||
|
||||
var match = list
|
||||
.Where(e => e.ModelDate <= asOfDate)
|
||||
.OrderByDescending(e => e.ModelDate)
|
||||
.FirstOrDefault();
|
||||
|
||||
return Task.FromResult<EpssEvidence?>(match);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<EpssEvidence>> GetHistoryAsync(
|
||||
string cveId,
|
||||
DateOnly startDate,
|
||||
DateOnly endDate,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(cveId))
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<EpssEvidence>>(Array.Empty<EpssEvidence>());
|
||||
}
|
||||
|
||||
var key = NormalizeCveId(cveId);
|
||||
if (!_history.TryGetValue(key, out var list))
|
||||
{
|
||||
return Task.FromResult<IReadOnlyList<EpssEvidence>>(Array.Empty<EpssEvidence>());
|
||||
}
|
||||
|
||||
var filtered = list
|
||||
.Where(e => e.ModelDate >= startDate && e.ModelDate <= endDate)
|
||||
.OrderBy(e => e.ModelDate)
|
||||
.ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<EpssEvidence>>(filtered);
|
||||
}
|
||||
|
||||
public Task<DateOnly?> GetLatestModelDateAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(LatestModelDate);
|
||||
|
||||
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(Available);
|
||||
|
||||
public void SetCurrent(EpssEvidence evidence)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(evidence);
|
||||
_current[NormalizeCveId(evidence.CveId)] = evidence;
|
||||
}
|
||||
|
||||
public void SetHistory(string cveId, IEnumerable<EpssEvidence> history)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(cveId);
|
||||
ArgumentNullException.ThrowIfNull(history);
|
||||
_history[NormalizeCveId(cveId)] = history
|
||||
.OrderBy(e => e.ModelDate)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string NormalizeCveId(string cveId)
|
||||
=> cveId.Trim().ToUpperInvariant();
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
// FidelityMetricsIntegrationTests.cs
|
||||
// Sprint: SPRINT_3403_0001_0001_fidelity_metrics
|
||||
// Task: FID-3403-013
|
||||
// Description: Integration tests for fidelity metrics in determinism harness
|
||||
// Description: Integration tests for fidelity metrics in determinism reports
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StellaOps.Scanner.Worker.Determinism;
|
||||
@@ -16,13 +16,12 @@ public sealed class FidelityMetricsIntegrationTests
|
||||
[Fact]
|
||||
public void DeterminismReport_WithFidelityMetrics_IncludesAllThreeTiers()
|
||||
{
|
||||
// Arrange & Act
|
||||
var fidelity = CreateTestFidelityMetrics(
|
||||
bitwiseFidelity: 0.98,
|
||||
semanticFidelity: 0.99,
|
||||
policyFidelity: 1.0);
|
||||
|
||||
var report = new DeterminismReport(
|
||||
var report = new global::StellaOps.Scanner.Worker.Determinism.DeterminismReport(
|
||||
Version: "1.0.0",
|
||||
Release: "test-release",
|
||||
Platform: "linux-amd64",
|
||||
@@ -35,9 +34,8 @@ public sealed class FidelityMetricsIntegrationTests
|
||||
Images: [],
|
||||
Fidelity: fidelity);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(report.Fidelity);
|
||||
Assert.Equal(0.98, report.Fidelity.BitwiseFidelity);
|
||||
Assert.Equal(0.98, report.Fidelity!.BitwiseFidelity);
|
||||
Assert.Equal(0.99, report.Fidelity.SemanticFidelity);
|
||||
Assert.Equal(1.0, report.Fidelity.PolicyFidelity);
|
||||
}
|
||||
@@ -45,13 +43,12 @@ public sealed class FidelityMetricsIntegrationTests
|
||||
[Fact]
|
||||
public void DeterminismImageReport_WithFidelityMetrics_TracksPerImage()
|
||||
{
|
||||
// Arrange
|
||||
var imageFidelity = CreateTestFidelityMetrics(
|
||||
bitwiseFidelity: 0.95,
|
||||
semanticFidelity: 0.98,
|
||||
policyFidelity: 1.0);
|
||||
|
||||
var imageReport = new DeterminismImageReport(
|
||||
var imageReport = new global::StellaOps.Scanner.Worker.Determinism.DeterminismImageReport(
|
||||
Image: "sha256:image123",
|
||||
Runs: 5,
|
||||
Identical: 4,
|
||||
@@ -60,120 +57,40 @@ public sealed class FidelityMetricsIntegrationTests
|
||||
RunsDetail: [],
|
||||
Fidelity: imageFidelity);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(imageReport.Fidelity);
|
||||
Assert.Equal(0.95, imageReport.Fidelity.BitwiseFidelity);
|
||||
Assert.Equal(0.95, imageReport.Fidelity!.BitwiseFidelity);
|
||||
Assert.Equal(5, imageReport.Fidelity.TotalReplays);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FidelityMetricsService_ComputesAllThreeTiers()
|
||||
public void FidelityMetricsService_Calculate_ComputesAllThreeTiers()
|
||||
{
|
||||
// Arrange
|
||||
var service = new FidelityMetricsService(
|
||||
new BitwiseFidelityCalculator(),
|
||||
new SemanticFidelityCalculator(),
|
||||
new PolicyFidelityCalculator());
|
||||
var service = new FidelityMetricsService();
|
||||
|
||||
var baseline = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "pass");
|
||||
var replay = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "pass");
|
||||
|
||||
// Act
|
||||
var metrics = service.Compute(baseline, new[] { replay });
|
||||
|
||||
// Assert
|
||||
Assert.Equal(1, metrics.TotalReplays);
|
||||
Assert.True(metrics.BitwiseFidelity >= 0.0 && metrics.BitwiseFidelity <= 1.0);
|
||||
Assert.True(metrics.SemanticFidelity >= 0.0 && metrics.SemanticFidelity <= 1.0);
|
||||
Assert.True(metrics.PolicyFidelity >= 0.0 && metrics.PolicyFidelity <= 1.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FidelityMetrics_SemanticEquivalent_ButBitwiseDifferent()
|
||||
{
|
||||
// Arrange - same semantic content, different formatting/ordering
|
||||
var service = new FidelityMetricsService(
|
||||
new BitwiseFidelityCalculator(),
|
||||
new SemanticFidelityCalculator(),
|
||||
new PolicyFidelityCalculator());
|
||||
|
||||
var baseline = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "HIGH", "pass");
|
||||
var replay = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "pass"); // case difference
|
||||
|
||||
// Act
|
||||
var metrics = service.Compute(baseline, new[] { replay });
|
||||
|
||||
// Assert
|
||||
// Bitwise should be < 1.0 (different bytes)
|
||||
// Semantic should be 1.0 (same meaning)
|
||||
// Policy should be 1.0 (same decision)
|
||||
Assert.True(metrics.SemanticFidelity >= metrics.BitwiseFidelity);
|
||||
Assert.Equal(1.0, metrics.PolicyFidelity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FidelityMetrics_PolicyDifference_ReflectedInPF()
|
||||
{
|
||||
// Arrange
|
||||
var service = new FidelityMetricsService(
|
||||
new BitwiseFidelityCalculator(),
|
||||
new SemanticFidelityCalculator(),
|
||||
new PolicyFidelityCalculator());
|
||||
|
||||
var baseline = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "pass");
|
||||
var replay = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "fail"); // policy differs
|
||||
|
||||
// Act
|
||||
var metrics = service.Compute(baseline, new[] { replay });
|
||||
|
||||
// Assert
|
||||
Assert.True(metrics.PolicyFidelity < 1.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FidelityMetrics_MultipleReplays_AveragesCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var service = new FidelityMetricsService(
|
||||
new BitwiseFidelityCalculator(),
|
||||
new SemanticFidelityCalculator(),
|
||||
new PolicyFidelityCalculator());
|
||||
|
||||
var baseline = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "pass");
|
||||
var replays = new[]
|
||||
var baselineHashes = new Dictionary<string, string>
|
||||
{
|
||||
CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "pass"), // identical
|
||||
CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "pass"), // identical
|
||||
CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "fail"), // policy diff
|
||||
["sbom.json"] = "sha256:baseline",
|
||||
};
|
||||
var replayHashes = new List<IReadOnlyDictionary<string, string>>
|
||||
{
|
||||
new Dictionary<string, string> { ["sbom.json"] = "sha256:baseline" }
|
||||
};
|
||||
|
||||
// Act
|
||||
var metrics = service.Compute(baseline, replays);
|
||||
var baselineFindings = CreateNormalizedFindings();
|
||||
var replayFindings = new List<NormalizedFindings> { CreateNormalizedFindings() };
|
||||
|
||||
// Assert
|
||||
Assert.Equal(3, metrics.TotalReplays);
|
||||
// 2 out of 3 have matching policy
|
||||
Assert.True(metrics.PolicyFidelity >= 0.6 && metrics.PolicyFidelity <= 0.7);
|
||||
}
|
||||
var baselineDecision = CreatePolicyDecision();
|
||||
var replayDecisions = new List<PolicyDecision> { CreatePolicyDecision() };
|
||||
|
||||
[Fact]
|
||||
public void FidelityMetrics_IncludesMismatchDiagnostics()
|
||||
{
|
||||
// Arrange
|
||||
var service = new FidelityMetricsService(
|
||||
new BitwiseFidelityCalculator(),
|
||||
new SemanticFidelityCalculator(),
|
||||
new PolicyFidelityCalculator());
|
||||
var metrics = service.Calculate(
|
||||
baselineHashes, replayHashes,
|
||||
baselineFindings, replayFindings,
|
||||
baselineDecision, replayDecisions);
|
||||
|
||||
var baseline = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "high", "pass");
|
||||
var replay = CreateTestScanResult("pkg:npm/lodash@4.17.21", "CVE-2021-23337", "critical", "fail"); // semantic + policy diff
|
||||
|
||||
// Act
|
||||
var metrics = service.Compute(baseline, new[] { replay });
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(metrics.Mismatches);
|
||||
Assert.NotEmpty(metrics.Mismatches);
|
||||
Assert.Equal(1, metrics.TotalReplays);
|
||||
Assert.Equal(1.0, metrics.BitwiseFidelity);
|
||||
Assert.Equal(1.0, metrics.SemanticFidelity);
|
||||
Assert.Equal(1.0, metrics.PolicyFidelity);
|
||||
}
|
||||
|
||||
private static FidelityMetrics CreateTestFidelityMetrics(
|
||||
@@ -195,38 +112,22 @@ public sealed class FidelityMetricsIntegrationTests
|
||||
};
|
||||
}
|
||||
|
||||
private static TestScanResult CreateTestScanResult(
|
||||
string purl,
|
||||
string cve,
|
||||
string severity,
|
||||
string policyDecision)
|
||||
private static NormalizedFindings CreateNormalizedFindings() => new()
|
||||
{
|
||||
return new TestScanResult
|
||||
Packages = new List<NormalizedPackage>
|
||||
{
|
||||
Packages = new[] { new TestPackage { Purl = purl } },
|
||||
Findings = new[] { new TestFinding { Cve = cve, Severity = severity } },
|
||||
PolicyDecision = policyDecision,
|
||||
PolicyReasonCodes = policyDecision == "pass" ? Array.Empty<string>() : new[] { "severity_exceeded" }
|
||||
};
|
||||
}
|
||||
new("pkg:npm/test@1.0.0", "1.0.0")
|
||||
},
|
||||
Cves = new HashSet<string> { "CVE-2024-0001" },
|
||||
SeverityCounts = new Dictionary<string, int> { ["MEDIUM"] = 1 },
|
||||
Verdicts = new Dictionary<string, string> { ["overall"] = "pass" }
|
||||
};
|
||||
|
||||
// Test support types
|
||||
private sealed record TestScanResult
|
||||
private static PolicyDecision CreatePolicyDecision() => new()
|
||||
{
|
||||
public required IReadOnlyList<TestPackage> Packages { get; init; }
|
||||
public required IReadOnlyList<TestFinding> Findings { get; init; }
|
||||
public required string PolicyDecision { get; init; }
|
||||
public required IReadOnlyList<string> PolicyReasonCodes { get; init; }
|
||||
}
|
||||
|
||||
private sealed record TestPackage
|
||||
{
|
||||
public required string Purl { get; init; }
|
||||
}
|
||||
|
||||
private sealed record TestFinding
|
||||
{
|
||||
public required string Cve { get; init; }
|
||||
public required string Severity { get; init; }
|
||||
}
|
||||
Passed = true,
|
||||
ReasonCodes = new List<string> { "CLEAN" },
|
||||
ViolationCount = 0,
|
||||
BlockLevel = "none"
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StellaOps.Scanner.Core.Epss;
|
||||
using StellaOps.Scanner.Storage.Epss;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests.Epss;
|
||||
|
||||
public sealed class EpssEnrichmentJobTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task EnrichAsync_EmitsPriorityChangedSignalWhenBandChanges()
|
||||
{
|
||||
var modelDate = new DateOnly(2027, 1, 16);
|
||||
|
||||
var changes = new List<EpssChangeRecord>
|
||||
{
|
||||
new()
|
||||
{
|
||||
CveId = "CVE-2024-0001",
|
||||
Flags = EpssChangeFlags.BigJumpUp,
|
||||
PreviousScore = 0.20,
|
||||
NewScore = 0.70,
|
||||
NewPercentile = 0.995,
|
||||
PreviousBand = EpssPriorityBand.Medium,
|
||||
ModelDate = modelDate
|
||||
}
|
||||
};
|
||||
|
||||
var epssRepository = new Mock<IEpssRepository>(MockBehavior.Strict);
|
||||
epssRepository
|
||||
.Setup(r => r.GetChangesAsync(modelDate, null, 100000, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(changes);
|
||||
|
||||
var epssProvider = new Mock<IEpssProvider>(MockBehavior.Strict);
|
||||
epssProvider
|
||||
.Setup(p => p.GetLatestModelDateAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(modelDate);
|
||||
epssProvider
|
||||
.Setup(p => p.GetCurrentBatchAsync(It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new EpssBatchResult
|
||||
{
|
||||
ModelDate = modelDate,
|
||||
Found = new[]
|
||||
{
|
||||
EpssEvidence.CreateWithTimestamp(
|
||||
"CVE-2024-0001",
|
||||
score: 0.70,
|
||||
percentile: 0.995,
|
||||
modelDate: modelDate,
|
||||
capturedAt: DateTimeOffset.Parse("2027-01-16T00:07:00Z"),
|
||||
source: "test",
|
||||
fromCache: false)
|
||||
},
|
||||
NotFound = Array.Empty<string>(),
|
||||
PartiallyFromCache = false,
|
||||
LookupTimeMs = 1
|
||||
});
|
||||
|
||||
var published = new List<(string cve, string oldBand, string newBand)>();
|
||||
var publisher = new Mock<IEpssSignalPublisher>(MockBehavior.Strict);
|
||||
publisher
|
||||
.Setup(p => p.PublishPriorityChangedAsync(
|
||||
It.IsAny<Guid>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<double>(),
|
||||
It.IsAny<DateOnly>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.Callback<Guid, string, string, string, double, DateOnly, CancellationToken>((_, cve, oldBand, newBand, _, _, _) =>
|
||||
published.Add((cve, oldBand, newBand)))
|
||||
.ReturnsAsync(new EpssSignalPublishResult { Success = true, MessageId = "ok" });
|
||||
|
||||
var job = new EpssEnrichmentJob(
|
||||
epssRepository.Object,
|
||||
epssProvider.Object,
|
||||
publisher.Object,
|
||||
Microsoft.Extensions.Options.Options.Create(new EpssEnrichmentOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BatchSize = 100,
|
||||
FlagsToProcess = EpssChangeFlags.None,
|
||||
HighPercentile = 0.99,
|
||||
CriticalPercentile = 0.995,
|
||||
MediumPercentile = 0.90,
|
||||
}),
|
||||
TimeProvider.System,
|
||||
NullLogger<EpssEnrichmentJob>.Instance);
|
||||
|
||||
await job.EnrichAsync();
|
||||
|
||||
Assert.Single(published);
|
||||
Assert.Equal("CVE-2024-0001", published[0].cve);
|
||||
Assert.Equal(EpssPriorityBand.Medium.ToString(), published[0].oldBand);
|
||||
Assert.Equal(EpssPriorityBand.Critical.ToString(), published[0].newBand);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Npgsql;
|
||||
using StellaOps.Scanner.Core.Epss;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.Epss;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests.Epss;
|
||||
|
||||
[Collection("scanner-worker-postgres")]
|
||||
public sealed class EpssSignalFlowIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private readonly ScannerWorkerPostgresFixture _fixture;
|
||||
private ScannerDataSource _dataSource = null!;
|
||||
|
||||
public EpssSignalFlowIntegrationTests(ScannerWorkerPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.TruncateAllTablesAsync();
|
||||
|
||||
var options = new ScannerStorageOptions
|
||||
{
|
||||
Postgres = new StellaOps.Infrastructure.Postgres.Options.PostgresOptions
|
||||
{
|
||||
ConnectionString = _fixture.ConnectionString,
|
||||
SchemaName = _fixture.SchemaName
|
||||
}
|
||||
};
|
||||
|
||||
_dataSource = new ScannerDataSource(Microsoft.Extensions.Options.Options.Create(options), NullLogger<ScannerDataSource>.Instance);
|
||||
|
||||
await using var connection = new NpgsqlConnection(_fixture.ConnectionString);
|
||||
await connection.OpenAsync();
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = $"""
|
||||
CREATE TABLE IF NOT EXISTS {_fixture.SchemaName}.vuln_instance_triage (
|
||||
instance_id UUID PRIMARY KEY,
|
||||
tenant_id UUID NOT NULL,
|
||||
cve_id TEXT NOT NULL
|
||||
);
|
||||
""";
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateSignalsAsync_WritesSignalsPerObservedTenant()
|
||||
{
|
||||
var epssRepository = new PostgresEpssRepository(_dataSource);
|
||||
var signalRepository = new PostgresEpssSignalRepository(_dataSource);
|
||||
var observedCveRepository = new PostgresObservedCveRepository(_dataSource);
|
||||
|
||||
var day1 = new DateOnly(2027, 1, 15);
|
||||
var run1 = await epssRepository.BeginImportAsync(day1, "bundle://day1.csv.gz", DateTimeOffset.Parse("2027-01-15T00:05:00Z"), "sha256:day1");
|
||||
var write1 = await epssRepository.WriteSnapshotAsync(
|
||||
run1.ImportRunId,
|
||||
day1,
|
||||
DateTimeOffset.Parse("2027-01-15T00:06:00Z"),
|
||||
ToAsync(new[]
|
||||
{
|
||||
new EpssScoreRow("CVE-2024-0001", 0.40, 0.90),
|
||||
new EpssScoreRow("CVE-2024-0002", 0.60, 0.96)
|
||||
}));
|
||||
await epssRepository.MarkImportSucceededAsync(run1.ImportRunId, write1.RowCount, "sha256:decompressed1", "v2027.01.15", day1);
|
||||
|
||||
var day2 = new DateOnly(2027, 1, 16);
|
||||
var run2 = await epssRepository.BeginImportAsync(day2, "bundle://day2.csv.gz", DateTimeOffset.Parse("2027-01-16T00:05:00Z"), "sha256:day2");
|
||||
var write2 = await epssRepository.WriteSnapshotAsync(
|
||||
run2.ImportRunId,
|
||||
day2,
|
||||
DateTimeOffset.Parse("2027-01-16T00:06:00Z"),
|
||||
ToAsync(new[]
|
||||
{
|
||||
new EpssScoreRow("CVE-2024-0001", 0.55, 0.95),
|
||||
new EpssScoreRow("CVE-2024-0002", 0.45, 0.94),
|
||||
new EpssScoreRow("CVE-2024-0003", 0.70, 0.97)
|
||||
}));
|
||||
await epssRepository.MarkImportSucceededAsync(run2.ImportRunId, write2.RowCount, "sha256:decompressed2", "v2027.01.16", day2);
|
||||
|
||||
var tenantA = Guid.Parse("aaaaaaaa-1111-1111-1111-111111111111");
|
||||
var tenantB = Guid.Parse("bbbbbbbb-2222-2222-2222-222222222222");
|
||||
|
||||
await InsertTriageRowAsync(tenantA, Guid.Parse("00000000-0000-0000-0000-000000000001"), "CVE-2024-0001");
|
||||
await InsertTriageRowAsync(tenantA, Guid.Parse("00000000-0000-0000-0000-000000000002"), "CVE-2024-0003");
|
||||
await InsertTriageRowAsync(tenantB, Guid.Parse("00000000-0000-0000-0000-000000000003"), "CVE-2024-0002");
|
||||
|
||||
var provider = new FixedEpssProvider(day2);
|
||||
var publisher = new RecordingEpssSignalPublisher();
|
||||
|
||||
var job = new EpssSignalJob(
|
||||
epssRepository,
|
||||
signalRepository,
|
||||
observedCveRepository,
|
||||
publisher,
|
||||
provider,
|
||||
Microsoft.Extensions.Options.Options.Create(new EpssSignalOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BatchSize = 500
|
||||
}),
|
||||
TimeProvider.System,
|
||||
NullLogger<EpssSignalJob>.Instance);
|
||||
|
||||
await job.GenerateSignalsAsync();
|
||||
|
||||
var tenantASignals = await signalRepository.GetByTenantAsync(tenantA, day2, day2);
|
||||
Assert.Equal(2, tenantASignals.Count);
|
||||
Assert.Contains(tenantASignals, s => s.CveId == "CVE-2024-0001");
|
||||
Assert.Contains(tenantASignals, s => s.CveId == "CVE-2024-0003");
|
||||
|
||||
var tenantBSignals = await signalRepository.GetByTenantAsync(tenantB, day2, day2);
|
||||
Assert.Single(tenantBSignals);
|
||||
Assert.Equal("CVE-2024-0002", tenantBSignals[0].CveId);
|
||||
|
||||
Assert.Equal(3, publisher.Published.Count);
|
||||
Assert.All(publisher.Published, s => Assert.Equal(day2, s.ModelDate));
|
||||
}
|
||||
|
||||
private async Task InsertTriageRowAsync(Guid tenantId, Guid instanceId, string cveId)
|
||||
{
|
||||
await using var connection = new NpgsqlConnection(_fixture.ConnectionString);
|
||||
await connection.OpenAsync();
|
||||
await using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = $"""
|
||||
INSERT INTO {_fixture.SchemaName}.vuln_instance_triage (instance_id, tenant_id, cve_id)
|
||||
VALUES (@InstanceId, @TenantId, @CveId)
|
||||
ON CONFLICT (instance_id) DO NOTHING;
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("InstanceId", instanceId);
|
||||
cmd.Parameters.AddWithValue("TenantId", tenantId);
|
||||
cmd.Parameters.AddWithValue("CveId", cveId);
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
private static async IAsyncEnumerable<EpssScoreRow> ToAsync(IEnumerable<EpssScoreRow> rows)
|
||||
{
|
||||
foreach (var row in rows)
|
||||
{
|
||||
yield return row;
|
||||
await Task.Yield();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FixedEpssProvider : IEpssProvider
|
||||
{
|
||||
private readonly DateOnly? _latestModelDate;
|
||||
|
||||
public FixedEpssProvider(DateOnly? latestModelDate)
|
||||
{
|
||||
_latestModelDate = latestModelDate;
|
||||
}
|
||||
|
||||
public Task<EpssEvidence?> GetCurrentAsync(string cveId, CancellationToken cancellationToken = default) => throw new NotSupportedException();
|
||||
public Task<EpssBatchResult> GetCurrentBatchAsync(IEnumerable<string> cveIds, CancellationToken cancellationToken = default) => throw new NotSupportedException();
|
||||
public Task<EpssEvidence?> GetAsOfDateAsync(string cveId, DateOnly asOfDate, CancellationToken cancellationToken = default) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<EpssEvidence>> GetHistoryAsync(string cveId, DateOnly startDate, DateOnly endDate, CancellationToken cancellationToken = default) => throw new NotSupportedException();
|
||||
public Task<DateOnly?> GetLatestModelDateAsync(CancellationToken cancellationToken = default) => Task.FromResult(_latestModelDate);
|
||||
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default) => Task.FromResult(true);
|
||||
}
|
||||
|
||||
private sealed class RecordingEpssSignalPublisher : IEpssSignalPublisher
|
||||
{
|
||||
public List<EpssSignal> Published { get; } = new();
|
||||
|
||||
public Task<EpssSignalPublishResult> PublishAsync(EpssSignal signal, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Published.Add(signal);
|
||||
return Task.FromResult(new EpssSignalPublishResult { Success = true, MessageId = "recorded" });
|
||||
}
|
||||
|
||||
public Task<int> PublishBatchAsync(IEnumerable<EpssSignal> signals, CancellationToken cancellationToken = default)
|
||||
{
|
||||
Published.AddRange(signals);
|
||||
return Task.FromResult(signals.Count());
|
||||
}
|
||||
|
||||
public Task<EpssSignalPublishResult> PublishPriorityChangedAsync(Guid tenantId, string cveId, string oldBand, string newBand, double epssScore, DateOnly modelDate, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new EpssSignalPublishResult { Success = true, MessageId = "recorded" });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StellaOps.Scanner.Core.Epss;
|
||||
using StellaOps.Scanner.Storage.Epss;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests.Epss;
|
||||
|
||||
public sealed class EpssSignalJobTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task GenerateSignalsAsync_CreatesSignalsAndPublishesBatch()
|
||||
{
|
||||
var modelDate = new DateOnly(2027, 1, 16);
|
||||
var tenantId = Guid.Parse("11111111-1111-1111-1111-111111111111");
|
||||
|
||||
var provider = new FixedEpssProvider(modelDate);
|
||||
|
||||
var epssRepository = new Mock<IEpssRepository>(MockBehavior.Strict);
|
||||
epssRepository
|
||||
.Setup(r => r.GetImportRunAsync(modelDate, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new EpssImportRun(
|
||||
ImportRunId: Guid.Parse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"),
|
||||
ModelDate: modelDate,
|
||||
SourceUri: "bundle://test.csv.gz",
|
||||
RetrievedAtUtc: DateTimeOffset.Parse("2027-01-16T00:05:00Z"),
|
||||
FileSha256: "sha256:test",
|
||||
DecompressedSha256: "sha256:decompressed",
|
||||
RowCount: 3,
|
||||
ModelVersionTag: "v2027.01.16",
|
||||
PublishedDate: modelDate,
|
||||
Status: "SUCCEEDED",
|
||||
Error: null,
|
||||
CreatedAtUtc: DateTimeOffset.Parse("2027-01-16T00:06:00Z")));
|
||||
|
||||
var changes = new List<EpssChangeRecord>
|
||||
{
|
||||
new()
|
||||
{
|
||||
CveId = "CVE-2024-0001",
|
||||
Flags = EpssChangeFlags.BigJumpUp,
|
||||
PreviousScore = 0.10,
|
||||
NewScore = 0.30,
|
||||
NewPercentile = 0.995,
|
||||
PreviousBand = EpssPriorityBand.Medium,
|
||||
ModelDate = modelDate
|
||||
},
|
||||
new()
|
||||
{
|
||||
CveId = "CVE-2024-0002",
|
||||
Flags = EpssChangeFlags.NewScored,
|
||||
PreviousScore = null,
|
||||
NewScore = 0.60,
|
||||
NewPercentile = 0.97,
|
||||
PreviousBand = EpssPriorityBand.Unknown,
|
||||
ModelDate = modelDate
|
||||
}
|
||||
};
|
||||
|
||||
epssRepository
|
||||
.Setup(r => r.GetChangesAsync(modelDate, null, 200000, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(changes);
|
||||
|
||||
var observedCveRepository = new Mock<IObservedCveRepository>(MockBehavior.Strict);
|
||||
observedCveRepository
|
||||
.Setup(r => r.GetActiveTenantsAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new[] { tenantId });
|
||||
observedCveRepository
|
||||
.Setup(r => r.FilterObservedAsync(tenantId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((Guid _, IEnumerable<string> cves, CancellationToken __) =>
|
||||
new HashSet<string>(cves, StringComparer.OrdinalIgnoreCase));
|
||||
|
||||
var createdSignals = new List<EpssSignal>();
|
||||
var signalRepository = new Mock<IEpssSignalRepository>(MockBehavior.Strict);
|
||||
signalRepository
|
||||
.Setup(r => r.CreateBulkAsync(It.IsAny<IEnumerable<EpssSignal>>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<IEnumerable<EpssSignal>, CancellationToken>((signals, _) => createdSignals.AddRange(signals))
|
||||
.ReturnsAsync((IEnumerable<EpssSignal> signals, CancellationToken _) => signals.Count());
|
||||
signalRepository
|
||||
.Setup(r => r.CreateAsync(It.IsAny<EpssSignal>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((EpssSignal signal, CancellationToken _) => signal);
|
||||
signalRepository
|
||||
.Setup(r => r.PruneAsync(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(0);
|
||||
signalRepository
|
||||
.Setup(r => r.GetByTenantAsync(It.IsAny<Guid>(), It.IsAny<DateOnly>(), It.IsAny<DateOnly>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Array.Empty<EpssSignal>());
|
||||
signalRepository
|
||||
.Setup(r => r.GetByCveAsync(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Array.Empty<EpssSignal>());
|
||||
signalRepository
|
||||
.Setup(r => r.GetHighPriorityAsync(It.IsAny<Guid>(), It.IsAny<DateOnly>(), It.IsAny<DateOnly>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Array.Empty<EpssSignal>());
|
||||
signalRepository
|
||||
.Setup(r => r.GetConfigAsync(It.IsAny<Guid>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((EpssSignalConfig?)null);
|
||||
signalRepository
|
||||
.Setup(r => r.UpsertConfigAsync(It.IsAny<EpssSignalConfig>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((EpssSignalConfig cfg, CancellationToken _) => cfg);
|
||||
|
||||
var publisher = new Mock<IEpssSignalPublisher>(MockBehavior.Strict);
|
||||
publisher
|
||||
.Setup(p => p.PublishBatchAsync(It.IsAny<IEnumerable<EpssSignal>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((IEnumerable<EpssSignal> signals, CancellationToken _) => signals.Count());
|
||||
publisher
|
||||
.Setup(p => p.PublishAsync(It.IsAny<EpssSignal>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new EpssSignalPublishResult { Success = true, MessageId = "ok" });
|
||||
publisher
|
||||
.Setup(p => p.PublishPriorityChangedAsync(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<double>(), It.IsAny<DateOnly>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new EpssSignalPublishResult { Success = true, MessageId = "ok" });
|
||||
|
||||
var job = new EpssSignalJob(
|
||||
epssRepository.Object,
|
||||
signalRepository.Object,
|
||||
observedCveRepository.Object,
|
||||
publisher.Object,
|
||||
provider,
|
||||
Microsoft.Extensions.Options.Options.Create(new EpssSignalOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BatchSize = 500
|
||||
}),
|
||||
TimeProvider.System,
|
||||
NullLogger<EpssSignalJob>.Instance);
|
||||
|
||||
await job.GenerateSignalsAsync();
|
||||
|
||||
Assert.Equal(2, createdSignals.Count);
|
||||
Assert.All(createdSignals, s =>
|
||||
{
|
||||
Assert.Equal(tenantId, s.TenantId);
|
||||
Assert.Equal(modelDate, s.ModelDate);
|
||||
Assert.Equal("v2027.01.16", s.ModelVersion);
|
||||
Assert.False(s.IsModelChange);
|
||||
Assert.False(string.IsNullOrWhiteSpace(s.DedupeKey));
|
||||
Assert.NotNull(s.ExplainHash);
|
||||
Assert.NotEmpty(s.ExplainHash);
|
||||
});
|
||||
|
||||
Assert.Contains(createdSignals, s => s.EventType == EpssSignalEventTypes.NewHigh && s.CveId == "CVE-2024-0002");
|
||||
Assert.Contains(createdSignals, s => s.EventType == EpssSignalEventTypes.RiskSpike && s.CveId == "CVE-2024-0001");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateSignalsAsync_EmitsModelUpdatedSummarySignal()
|
||||
{
|
||||
var modelDate = new DateOnly(2027, 1, 16);
|
||||
var tenantId = Guid.Parse("22222222-2222-2222-2222-222222222222");
|
||||
|
||||
var provider = new FixedEpssProvider(modelDate);
|
||||
|
||||
var epssRepository = new Mock<IEpssRepository>(MockBehavior.Strict);
|
||||
epssRepository
|
||||
.SetupSequence(r => r.GetImportRunAsync(modelDate, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new EpssImportRun(
|
||||
ImportRunId: Guid.Parse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"),
|
||||
ModelDate: modelDate,
|
||||
SourceUri: "bundle://test.csv.gz",
|
||||
RetrievedAtUtc: DateTimeOffset.Parse("2027-01-16T00:05:00Z"),
|
||||
FileSha256: "sha256:test",
|
||||
DecompressedSha256: "sha256:decompressed",
|
||||
RowCount: 1,
|
||||
ModelVersionTag: "v2027.01.16",
|
||||
PublishedDate: modelDate,
|
||||
Status: "SUCCEEDED",
|
||||
Error: null,
|
||||
CreatedAtUtc: DateTimeOffset.Parse("2027-01-16T00:06:00Z")))
|
||||
.ReturnsAsync(new EpssImportRun(
|
||||
ImportRunId: Guid.Parse("cccccccc-cccc-cccc-cccc-cccccccccccc"),
|
||||
ModelDate: modelDate,
|
||||
SourceUri: "bundle://test.csv.gz",
|
||||
RetrievedAtUtc: DateTimeOffset.Parse("2027-01-16T00:05:00Z"),
|
||||
FileSha256: "sha256:test",
|
||||
DecompressedSha256: "sha256:decompressed",
|
||||
RowCount: 1,
|
||||
ModelVersionTag: "v2027.01.16b",
|
||||
PublishedDate: modelDate,
|
||||
Status: "SUCCEEDED",
|
||||
Error: null,
|
||||
CreatedAtUtc: DateTimeOffset.Parse("2027-01-16T00:06:00Z")));
|
||||
|
||||
var changes = new List<EpssChangeRecord>
|
||||
{
|
||||
new()
|
||||
{
|
||||
CveId = "CVE-2024-0001",
|
||||
Flags = EpssChangeFlags.NewScored,
|
||||
PreviousScore = null,
|
||||
NewScore = 0.10,
|
||||
NewPercentile = 0.91,
|
||||
PreviousBand = EpssPriorityBand.Unknown,
|
||||
ModelDate = modelDate
|
||||
}
|
||||
};
|
||||
|
||||
epssRepository
|
||||
.Setup(r => r.GetChangesAsync(modelDate, null, 200000, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(changes);
|
||||
|
||||
var observedCveRepository = new Mock<IObservedCveRepository>(MockBehavior.Strict);
|
||||
observedCveRepository
|
||||
.Setup(r => r.GetActiveTenantsAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new[] { tenantId });
|
||||
observedCveRepository
|
||||
.Setup(r => r.FilterObservedAsync(tenantId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((Guid _, IEnumerable<string> cves, CancellationToken __) =>
|
||||
new HashSet<string>(cves, StringComparer.OrdinalIgnoreCase));
|
||||
|
||||
var createdSignals = new List<EpssSignal>();
|
||||
var createdSummaries = new List<EpssSignal>();
|
||||
|
||||
var signalRepository = new Mock<IEpssSignalRepository>(MockBehavior.Strict);
|
||||
signalRepository
|
||||
.Setup(r => r.CreateBulkAsync(It.IsAny<IEnumerable<EpssSignal>>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<IEnumerable<EpssSignal>, CancellationToken>((signals, _) => createdSignals.AddRange(signals))
|
||||
.ReturnsAsync((IEnumerable<EpssSignal> signals, CancellationToken _) => signals.Count());
|
||||
signalRepository
|
||||
.Setup(r => r.CreateAsync(It.IsAny<EpssSignal>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<EpssSignal, CancellationToken>((signal, _) => createdSummaries.Add(signal))
|
||||
.ReturnsAsync((EpssSignal signal, CancellationToken _) => signal);
|
||||
signalRepository
|
||||
.Setup(r => r.PruneAsync(It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(0);
|
||||
signalRepository
|
||||
.Setup(r => r.GetByTenantAsync(It.IsAny<Guid>(), It.IsAny<DateOnly>(), It.IsAny<DateOnly>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Array.Empty<EpssSignal>());
|
||||
signalRepository
|
||||
.Setup(r => r.GetByCveAsync(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Array.Empty<EpssSignal>());
|
||||
signalRepository
|
||||
.Setup(r => r.GetHighPriorityAsync(It.IsAny<Guid>(), It.IsAny<DateOnly>(), It.IsAny<DateOnly>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Array.Empty<EpssSignal>());
|
||||
signalRepository
|
||||
.Setup(r => r.GetConfigAsync(It.IsAny<Guid>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((EpssSignalConfig?)null);
|
||||
signalRepository
|
||||
.Setup(r => r.UpsertConfigAsync(It.IsAny<EpssSignalConfig>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((EpssSignalConfig cfg, CancellationToken _) => cfg);
|
||||
|
||||
var publisher = new Mock<IEpssSignalPublisher>(MockBehavior.Strict);
|
||||
publisher
|
||||
.Setup(p => p.PublishBatchAsync(It.IsAny<IEnumerable<EpssSignal>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((IEnumerable<EpssSignal> signals, CancellationToken _) => signals.Count());
|
||||
publisher
|
||||
.Setup(p => p.PublishAsync(It.IsAny<EpssSignal>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new EpssSignalPublishResult { Success = true, MessageId = "ok" });
|
||||
publisher
|
||||
.Setup(p => p.PublishPriorityChangedAsync(It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<double>(), It.IsAny<DateOnly>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new EpssSignalPublishResult { Success = true, MessageId = "ok" });
|
||||
|
||||
var job = new EpssSignalJob(
|
||||
epssRepository.Object,
|
||||
signalRepository.Object,
|
||||
observedCveRepository.Object,
|
||||
publisher.Object,
|
||||
provider,
|
||||
Microsoft.Extensions.Options.Options.Create(new EpssSignalOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BatchSize = 500
|
||||
}),
|
||||
TimeProvider.System,
|
||||
NullLogger<EpssSignalJob>.Instance);
|
||||
|
||||
await job.GenerateSignalsAsync(); // establishes _lastModelVersion
|
||||
await job.GenerateSignalsAsync(); // model version changes -> emits summary
|
||||
|
||||
Assert.Single(createdSummaries);
|
||||
Assert.Equal(EpssSignalEventTypes.ModelUpdated, createdSummaries[0].EventType);
|
||||
Assert.Equal("MODEL_UPDATE", createdSummaries[0].CveId);
|
||||
Assert.True(createdSummaries[0].IsModelChange);
|
||||
Assert.Contains("v2027.01.16->v2027.01.16b", createdSummaries[0].DedupeKey, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private sealed class FixedEpssProvider : IEpssProvider
|
||||
{
|
||||
private readonly DateOnly? _latestModelDate;
|
||||
|
||||
public FixedEpssProvider(DateOnly? latestModelDate)
|
||||
{
|
||||
_latestModelDate = latestModelDate;
|
||||
}
|
||||
|
||||
public Task<EpssEvidence?> GetCurrentAsync(string cveId, CancellationToken cancellationToken = default) => throw new NotSupportedException();
|
||||
public Task<EpssBatchResult> GetCurrentBatchAsync(IEnumerable<string> cveIds, CancellationToken cancellationToken = default) => throw new NotSupportedException();
|
||||
public Task<EpssEvidence?> GetAsOfDateAsync(string cveId, DateOnly asOfDate, CancellationToken cancellationToken = default) => throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<EpssEvidence>> GetHistoryAsync(string cveId, DateOnly startDate, DateOnly endDate, CancellationToken cancellationToken = default) => throw new NotSupportedException();
|
||||
public Task<DateOnly?> GetLatestModelDateAsync(CancellationToken cancellationToken = default) => Task.FromResult(_latestModelDate);
|
||||
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default) => Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using System.Reflection;
|
||||
using StellaOps.Infrastructure.Postgres.Testing;
|
||||
using StellaOps.Scanner.Storage;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests.Epss;
|
||||
|
||||
public sealed class ScannerWorkerPostgresFixture : PostgresIntegrationFixture, ICollectionFixture<ScannerWorkerPostgresFixture>
|
||||
{
|
||||
protected override Assembly? GetMigrationAssembly() => typeof(ScannerStorageOptions).Assembly;
|
||||
|
||||
protected override string GetModuleName() => "Scanner.Storage";
|
||||
}
|
||||
|
||||
[CollectionDefinition("scanner-worker-postgres")]
|
||||
public sealed class ScannerWorkerPostgresCollection : ICollectionFixture<ScannerWorkerPostgresFixture>
|
||||
{
|
||||
}
|
||||
@@ -29,8 +29,8 @@ public sealed class ScanCompletionMetricsIntegrationTests
|
||||
.Callback<ScanMetrics, CancellationToken>((m, _) => savedMetrics.Add(m))
|
||||
.Returns(Task.CompletedTask);
|
||||
mockRepository
|
||||
.Setup(r => r.SavePhasesAsync(It.IsAny<IEnumerable<ExecutionPhase>>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<IEnumerable<ExecutionPhase>, CancellationToken>((p, _) => savedPhases.AddRange(p))
|
||||
.Setup(r => r.SavePhasesAsync(It.IsAny<IReadOnlyList<ExecutionPhase>>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<IReadOnlyList<ExecutionPhase>, CancellationToken>((p, _) => savedPhases.AddRange(p))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var factory = new TestScanMetricsCollectorFactory(mockRepository.Object);
|
||||
@@ -120,7 +120,7 @@ public sealed class ScanCompletionMetricsIntegrationTests
|
||||
.Callback<ScanMetrics, CancellationToken>((m, _) => savedMetrics.Add(m))
|
||||
.Returns(Task.CompletedTask);
|
||||
mockRepository
|
||||
.Setup(r => r.SavePhasesAsync(It.IsAny<IEnumerable<ExecutionPhase>>(), It.IsAny<CancellationToken>()))
|
||||
.Setup(r => r.SavePhasesAsync(It.IsAny<IReadOnlyList<ExecutionPhase>>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var factory = new TestScanMetricsCollectorFactory(mockRepository.Object);
|
||||
@@ -162,7 +162,7 @@ public sealed class ScanCompletionMetricsIntegrationTests
|
||||
.Callback<ScanMetrics, CancellationToken>((m, _) => savedMetrics.Add(m))
|
||||
.Returns(Task.CompletedTask);
|
||||
mockRepository
|
||||
.Setup(r => r.SavePhasesAsync(It.IsAny<IEnumerable<ExecutionPhase>>(), It.IsAny<CancellationToken>()))
|
||||
.Setup(r => r.SavePhasesAsync(It.IsAny<IReadOnlyList<ExecutionPhase>>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var factory = new TestScanMetricsCollectorFactory(mockRepository.Object);
|
||||
|
||||
@@ -11,5 +11,9 @@
|
||||
<ProjectReference Include="../../StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/StellaOps.Scanner.Analyzers.Lang.Ruby.csproj" />
|
||||
<ProjectReference Include="..\..\..\__Libraries\StellaOps.Infrastructure.Postgres.Testing\StellaOps.Infrastructure.Postgres.Testing.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Moq" Version="4.20.72" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user