feat(rate-limiting): Implement core rate limiting functionality with configuration, decision-making, metrics, middleware, and service registration
- Add RateLimitConfig for configuration management with YAML binding support. - Introduce RateLimitDecision to encapsulate the result of rate limit checks. - Implement RateLimitMetrics for OpenTelemetry metrics tracking. - Create RateLimitMiddleware for enforcing rate limits on incoming requests. - Develop RateLimitService to orchestrate instance and environment rate limit checks. - Add RateLimitServiceCollectionExtensions for dependency injection registration.
This commit is contained in:
master
@@ -0,0 +1,222 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// BenchmarkResultWriter.cs
|
||||
// Sprint: SPRINT_3500_0003_0001_ground_truth_corpus_ci_gates
|
||||
// Task: CORPUS-006 - Implement BenchmarkResultWriter with metrics calculation
|
||||
// Description: Writes benchmark results to JSON and computes metrics
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// Writes benchmark results to files and computes metrics.
|
||||
/// </summary>
|
||||
public interface IBenchmarkResultWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Write benchmark result to the results directory.
|
||||
/// </summary>
|
||||
Task WriteResultAsync(BenchmarkResult result, string outputPath, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Read the current baseline.
|
||||
/// </summary>
|
||||
Task<BenchmarkBaseline?> ReadBaselineAsync(string baselinePath, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Update the baseline from a benchmark result.
|
||||
/// </summary>
|
||||
Task UpdateBaselineAsync(BenchmarkResult result, string baselinePath, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Generate a markdown report from benchmark result.
|
||||
/// </summary>
|
||||
string GenerateMarkdownReport(BenchmarkResult result, BenchmarkBaseline? baseline = null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of IBenchmarkResultWriter.
|
||||
/// </summary>
|
||||
public sealed class BenchmarkResultWriter : IBenchmarkResultWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task WriteResultAsync(BenchmarkResult result, string outputPath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
ArgumentException.ThrowIfNullOrEmpty(outputPath);
|
||||
|
||||
// Ensure directory exists
|
||||
var dir = Path.GetDirectoryName(outputPath);
|
||||
if (!string.IsNullOrEmpty(dir))
|
||||
Directory.CreateDirectory(dir);
|
||||
|
||||
var json = JsonSerializer.Serialize(result, JsonOptions);
|
||||
await File.WriteAllTextAsync(outputPath, json, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<BenchmarkBaseline?> ReadBaselineAsync(string baselinePath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!File.Exists(baselinePath))
|
||||
return null;
|
||||
|
||||
var json = await File.ReadAllTextAsync(baselinePath, cancellationToken);
|
||||
return JsonSerializer.Deserialize<BenchmarkBaseline>(json, JsonOptions);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpdateBaselineAsync(BenchmarkResult result, string baselinePath, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(result);
|
||||
|
||||
var baseline = new BenchmarkBaseline(
|
||||
Version: result.CorpusVersion,
|
||||
Timestamp: result.Timestamp,
|
||||
Precision: result.Metrics.Precision,
|
||||
Recall: result.Metrics.Recall,
|
||||
F1: result.Metrics.F1,
|
||||
TtfrpP95Ms: result.Metrics.TtfrpP95Ms);
|
||||
|
||||
var dir = Path.GetDirectoryName(baselinePath);
|
||||
if (!string.IsNullOrEmpty(dir))
|
||||
Directory.CreateDirectory(dir);
|
||||
|
||||
var json = JsonSerializer.Serialize(baseline, JsonOptions);
|
||||
await File.WriteAllTextAsync(baselinePath, json, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string GenerateMarkdownReport(BenchmarkResult result, BenchmarkBaseline? baseline = null)
|
||||
{
|
||||
var sb = new System.Text.StringBuilder();
|
||||
|
||||
sb.AppendLine("# Reachability Benchmark Report");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($"**Run ID:** `{result.RunId}`");
|
||||
sb.AppendLine($"**Timestamp:** {result.Timestamp:yyyy-MM-dd HH:mm:ss} UTC");
|
||||
sb.AppendLine($"**Corpus Version:** {result.CorpusVersion}");
|
||||
sb.AppendLine($"**Scanner Version:** {result.ScannerVersion}");
|
||||
sb.AppendLine($"**Duration:** {result.DurationMs}ms");
|
||||
sb.AppendLine();
|
||||
|
||||
sb.AppendLine("## Metrics Summary");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine("| Metric | Value | Baseline | Delta |");
|
||||
sb.AppendLine("|--------|-------|----------|-------|");
|
||||
|
||||
var m = result.Metrics;
|
||||
var b = baseline;
|
||||
|
||||
AppendMetricRow(sb, "Precision", m.Precision, b?.Precision);
|
||||
AppendMetricRow(sb, "Recall", m.Recall, b?.Recall);
|
||||
AppendMetricRow(sb, "F1 Score", m.F1, b?.F1);
|
||||
AppendMetricRow(sb, "TTFRP p50 (ms)", m.TtfrpP50Ms, null);
|
||||
AppendMetricRow(sb, "TTFRP p95 (ms)", m.TtfrpP95Ms, b?.TtfrpP95Ms);
|
||||
AppendMetricRow(sb, "Determinism", m.DeterministicReplay, null);
|
||||
|
||||
sb.AppendLine();
|
||||
|
||||
// Regression check
|
||||
if (baseline != null)
|
||||
{
|
||||
var check = result.CheckRegression(baseline);
|
||||
sb.AppendLine("## Regression Check");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine(check.Passed ? "✅ **PASSED**" : "❌ **FAILED**");
|
||||
sb.AppendLine();
|
||||
|
||||
if (check.Issues.Count > 0)
|
||||
{
|
||||
sb.AppendLine("### Issues");
|
||||
sb.AppendLine();
|
||||
foreach (var issue in check.Issues)
|
||||
{
|
||||
var icon = issue.Severity == RegressionSeverity.Error ? "🔴" : "🟡";
|
||||
sb.AppendLine($"- {icon} **{issue.Metric}**: {issue.Message}");
|
||||
}
|
||||
sb.AppendLine();
|
||||
}
|
||||
}
|
||||
|
||||
// Sample breakdown
|
||||
sb.AppendLine("## Sample Results");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine("| Sample | Category | Sinks | Correct | Latency | Deterministic |");
|
||||
sb.AppendLine("|--------|----------|-------|---------|---------|---------------|");
|
||||
|
||||
foreach (var sample in result.SampleResults)
|
||||
{
|
||||
var correct = sample.SinkResults.Count(s => s.Correct);
|
||||
var total = sample.SinkResults.Count;
|
||||
var status = correct == total ? "✅" : "❌";
|
||||
var detIcon = sample.Deterministic ? "✅" : "❌";
|
||||
|
||||
sb.AppendLine($"| {sample.SampleId} | {sample.Category} | {correct}/{total} {status} | {sample.LatencyMs}ms | {detIcon} |");
|
||||
}
|
||||
|
||||
// Failed sinks detail
|
||||
var failedSinks = result.SampleResults
|
||||
.SelectMany(s => s.SinkResults.Where(sink => !sink.Correct)
|
||||
.Select(sink => (s.SampleId, sink)))
|
||||
.ToList();
|
||||
|
||||
if (failedSinks.Count > 0)
|
||||
{
|
||||
sb.AppendLine();
|
||||
sb.AppendLine("## Failed Sinks");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine("| Sample | Sink | Expected | Actual |");
|
||||
sb.AppendLine("|--------|------|----------|--------|");
|
||||
|
||||
foreach (var (sampleId, sink) in failedSinks)
|
||||
{
|
||||
sb.AppendLine($"| {sampleId} | {sink.SinkId} | {sink.Expected} | {sink.Actual} |");
|
||||
}
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static void AppendMetricRow(System.Text.StringBuilder sb, string name, double value, double? baseline)
|
||||
{
|
||||
var formatted = name.Contains("ms") ? $"{value:N0}" : $"{value:P1}";
|
||||
var baselineStr = baseline.HasValue
|
||||
? (name.Contains("ms") ? $"{baseline.Value:N0}" : $"{baseline.Value:P1}")
|
||||
: "-";
|
||||
|
||||
string delta = "-";
|
||||
if (baseline.HasValue)
|
||||
{
|
||||
var diff = value - baseline.Value;
|
||||
var sign = diff >= 0 ? "+" : "";
|
||||
delta = name.Contains("ms")
|
||||
? $"{sign}{diff:N0}"
|
||||
: $"{sign}{diff:P1}";
|
||||
}
|
||||
|
||||
sb.AppendLine($"| {name} | {formatted} | {baselineStr} | {delta} |");
|
||||
}
|
||||
|
||||
private static void AppendMetricRow(System.Text.StringBuilder sb, string name, int value, int? baseline)
|
||||
{
|
||||
var baselineStr = baseline.HasValue ? $"{baseline.Value:N0}" : "-";
|
||||
string delta = "-";
|
||||
if (baseline.HasValue)
|
||||
{
|
||||
var diff = value - baseline.Value;
|
||||
var sign = diff >= 0 ? "+" : "";
|
||||
delta = $"{sign}{diff:N0}";
|
||||
}
|
||||
|
||||
sb.AppendLine($"| {name} | {value:N0} | {baselineStr} | {delta} |");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ICorpusRunner.cs
|
||||
// Sprint: SPRINT_3500_0003_0001_ground_truth_corpus_ci_gates
|
||||
// Task: CORPUS-005 - Implement ICorpusRunner interface for benchmark execution
|
||||
// Description: Interface and models for running ground-truth corpus benchmarks
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.Benchmarks;
|
||||
|
||||
/// <summary>
|
||||
/// Interface for running ground-truth corpus benchmarks.
|
||||
/// </summary>
|
||||
public interface ICorpusRunner
|
||||
{
|
||||
/// <summary>
|
||||
/// Run the full corpus and compute metrics.
|
||||
/// </summary>
|
||||
/// <param name="corpusPath">Path to corpus.json index file.</param>
|
||||
/// <param name="options">Run options.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Benchmark results with metrics.</returns>
|
||||
Task<BenchmarkResult> RunAsync(string corpusPath, CorpusRunOptions options, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Run a single sample from the corpus.
|
||||
/// </summary>
|
||||
/// <param name="samplePath">Path to sample.manifest.json.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Sample result.</returns>
|
||||
Task<SampleResult> RunSampleAsync(string samplePath, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options for corpus runs.
|
||||
/// </summary>
|
||||
public sealed record CorpusRunOptions
|
||||
{
|
||||
/// <summary>Filter to specific categories.</summary>
|
||||
public string[]? Categories { get; init; }
|
||||
|
||||
/// <summary>Filter to specific sample IDs.</summary>
|
||||
public string[]? SampleIds { get; init; }
|
||||
|
||||
/// <summary>Number of parallel workers.</summary>
|
||||
public int Parallelism { get; init; } = 1;
|
||||
|
||||
/// <summary>Timeout per sample in milliseconds.</summary>
|
||||
public int TimeoutMs { get; init; } = 30000;
|
||||
|
||||
/// <summary>Whether to run determinism checks.</summary>
|
||||
public bool CheckDeterminism { get; init; } = true;
|
||||
|
||||
/// <summary>Number of runs for determinism check.</summary>
|
||||
public int DeterminismRuns { get; init; } = 3;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a full benchmark run.
|
||||
/// </summary>
|
||||
public sealed record BenchmarkResult(
|
||||
[property: JsonPropertyName("runId")] string RunId,
|
||||
[property: JsonPropertyName("timestamp")] DateTimeOffset Timestamp,
|
||||
[property: JsonPropertyName("corpusVersion")] string CorpusVersion,
|
||||
[property: JsonPropertyName("scannerVersion")] string ScannerVersion,
|
||||
[property: JsonPropertyName("metrics")] BenchmarkMetrics Metrics,
|
||||
[property: JsonPropertyName("sampleResults")] IReadOnlyList<SampleResult> SampleResults,
|
||||
[property: JsonPropertyName("durationMs")] long DurationMs)
|
||||
{
|
||||
/// <summary>
|
||||
/// Check if the benchmark result meets the given thresholds.
|
||||
/// </summary>
|
||||
public RegressionCheckResult CheckRegression(BenchmarkBaseline baseline)
|
||||
{
|
||||
var issues = new List<RegressionIssue>();
|
||||
|
||||
// Precision check
|
||||
var precisionDrop = baseline.Precision - Metrics.Precision;
|
||||
if (precisionDrop > 0.01) // 1 percentage point
|
||||
{
|
||||
issues.Add(new RegressionIssue(
|
||||
"precision",
|
||||
$"Precision dropped from {baseline.Precision:P1} to {Metrics.Precision:P1} ({precisionDrop:P1})",
|
||||
RegressionSeverity.Error));
|
||||
}
|
||||
|
||||
// Recall check
|
||||
var recallDrop = baseline.Recall - Metrics.Recall;
|
||||
if (recallDrop > 0.01)
|
||||
{
|
||||
issues.Add(new RegressionIssue(
|
||||
"recall",
|
||||
$"Recall dropped from {baseline.Recall:P1} to {Metrics.Recall:P1} ({recallDrop:P1})",
|
||||
RegressionSeverity.Error));
|
||||
}
|
||||
|
||||
// Determinism check
|
||||
if (Metrics.DeterministicReplay < 1.0)
|
||||
{
|
||||
issues.Add(new RegressionIssue(
|
||||
"determinism",
|
||||
$"Deterministic replay is {Metrics.DeterministicReplay:P0} (expected 100%)",
|
||||
RegressionSeverity.Error));
|
||||
}
|
||||
|
||||
// TTFRP p95 check (warning only)
|
||||
var ttfrpIncrease = (Metrics.TtfrpP95Ms - baseline.TtfrpP95Ms) / (double)baseline.TtfrpP95Ms;
|
||||
if (ttfrpIncrease > 0.20)
|
||||
{
|
||||
issues.Add(new RegressionIssue(
|
||||
"ttfrp_p95",
|
||||
$"TTFRP p95 increased from {baseline.TtfrpP95Ms}ms to {Metrics.TtfrpP95Ms}ms ({ttfrpIncrease:P0})",
|
||||
RegressionSeverity.Warning));
|
||||
}
|
||||
|
||||
return new RegressionCheckResult(
|
||||
Passed: !issues.Any(i => i.Severity == RegressionSeverity.Error),
|
||||
Issues: issues);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Metrics from a benchmark run.
|
||||
/// </summary>
|
||||
public sealed record BenchmarkMetrics(
|
||||
[property: JsonPropertyName("precision")] double Precision,
|
||||
[property: JsonPropertyName("recall")] double Recall,
|
||||
[property: JsonPropertyName("f1")] double F1,
|
||||
[property: JsonPropertyName("ttfrp_p50_ms")] int TtfrpP50Ms,
|
||||
[property: JsonPropertyName("ttfrp_p95_ms")] int TtfrpP95Ms,
|
||||
[property: JsonPropertyName("deterministicReplay")] double DeterministicReplay)
|
||||
{
|
||||
public static BenchmarkMetrics Compute(IReadOnlyList<SampleResult> results)
|
||||
{
|
||||
if (results.Count == 0)
|
||||
return new(0, 0, 0, 0, 0, 1.0);
|
||||
|
||||
int tp = 0, fp = 0, tn = 0, fn = 0;
|
||||
var latencies = new List<int>();
|
||||
int deterministicCount = 0;
|
||||
|
||||
foreach (var r in results)
|
||||
{
|
||||
foreach (var sink in r.SinkResults)
|
||||
{
|
||||
if (sink.Expected == "reachable" && sink.Actual == "reachable") tp++;
|
||||
else if (sink.Expected == "reachable" && sink.Actual == "unreachable") fn++;
|
||||
else if (sink.Expected == "unreachable" && sink.Actual == "unreachable") tn++;
|
||||
else if (sink.Expected == "unreachable" && sink.Actual == "reachable") fp++;
|
||||
}
|
||||
|
||||
latencies.Add((int)r.LatencyMs);
|
||||
if (r.Deterministic) deterministicCount++;
|
||||
}
|
||||
|
||||
var precision = tp + fp > 0 ? (double)tp / (tp + fp) : 1.0;
|
||||
var recall = tp + fn > 0 ? (double)tp / (tp + fn) : 1.0;
|
||||
var f1 = precision + recall > 0 ? 2 * precision * recall / (precision + recall) : 0;
|
||||
|
||||
latencies.Sort();
|
||||
var p50 = latencies.Count > 0 ? latencies[latencies.Count / 2] : 0;
|
||||
var p95 = latencies.Count > 0 ? latencies[(int)(latencies.Count * 0.95)] : 0;
|
||||
|
||||
var determinism = results.Count > 0 ? (double)deterministicCount / results.Count : 1.0;
|
||||
|
||||
return new(
|
||||
Math.Round(precision, 4),
|
||||
Math.Round(recall, 4),
|
||||
Math.Round(f1, 4),
|
||||
p50,
|
||||
p95,
|
||||
determinism);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a single sample run.
|
||||
/// </summary>
|
||||
public sealed record SampleResult(
|
||||
[property: JsonPropertyName("sampleId")] string SampleId,
|
||||
[property: JsonPropertyName("name")] string Name,
|
||||
[property: JsonPropertyName("category")] string Category,
|
||||
[property: JsonPropertyName("sinkResults")] IReadOnlyList<SinkResult> SinkResults,
|
||||
[property: JsonPropertyName("latencyMs")] long LatencyMs,
|
||||
[property: JsonPropertyName("deterministic")] bool Deterministic,
|
||||
[property: JsonPropertyName("error")] string? Error = null);
|
||||
|
||||
/// <summary>
|
||||
/// Result for a single sink within a sample.
|
||||
/// </summary>
|
||||
public sealed record SinkResult(
|
||||
[property: JsonPropertyName("sinkId")] string SinkId,
|
||||
[property: JsonPropertyName("expected")] string Expected,
|
||||
[property: JsonPropertyName("actual")] string Actual,
|
||||
[property: JsonPropertyName("correct")] bool Correct,
|
||||
[property: JsonPropertyName("pathsFound")] IReadOnlyList<string[]>? PathsFound = null);
|
||||
|
||||
/// <summary>
|
||||
/// Baseline for regression checks.
|
||||
/// </summary>
|
||||
public sealed record BenchmarkBaseline(
|
||||
[property: JsonPropertyName("version")] string Version,
|
||||
[property: JsonPropertyName("timestamp")] DateTimeOffset Timestamp,
|
||||
[property: JsonPropertyName("precision")] double Precision,
|
||||
[property: JsonPropertyName("recall")] double Recall,
|
||||
[property: JsonPropertyName("f1")] double F1,
|
||||
[property: JsonPropertyName("ttfrp_p95_ms")] int TtfrpP95Ms);
|
||||
|
||||
/// <summary>
|
||||
/// Result of regression check.
|
||||
/// </summary>
|
||||
public sealed record RegressionCheckResult(
|
||||
bool Passed,
|
||||
IReadOnlyList<RegressionIssue> Issues);
|
||||
|
||||
/// <summary>
|
||||
/// A regression issue found during check.
|
||||
/// </summary>
|
||||
public sealed record RegressionIssue(
|
||||
string Metric,
|
||||
string Message,
|
||||
RegressionSeverity Severity);
|
||||
|
||||
/// <summary>
|
||||
/// Severity of a regression issue.
|
||||
/// </summary>
|
||||
public enum RegressionSeverity
|
||||
{
|
||||
Warning,
|
||||
Error
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>false</TreatWarningsAsErrors>
|
||||
<Description>Ground-truth corpus benchmarking infrastructure for reachability analysis</Description>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="System.Text.Json" Version="10.0.0-preview.1.25105.2" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Scanner.Reachability/StellaOps.Scanner.Reachability.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,255 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ProofBundleWriter.cs
|
||||
// Sprint: SPRINT_3401_0002_0001_score_replay_proof_bundle
|
||||
// Task: SCORE-REPLAY-008 - Implement ProofBundleWriter (ZIP + content-addressed)
|
||||
// Description: Creates content-addressed ZIP bundles with manifests and proofs
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Policy.Scoring;
|
||||
|
||||
namespace StellaOps.Scanner.Core;
|
||||
|
||||
/// <summary>
|
||||
/// Service for writing proof bundles to content-addressed storage.
|
||||
/// </summary>
|
||||
public interface IProofBundleWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Create a proof bundle containing the scan manifest and proof ledger.
|
||||
/// </summary>
|
||||
/// <param name="signedManifest">The signed scan manifest.</param>
|
||||
/// <param name="ledger">The proof ledger with all scoring nodes.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The proof bundle metadata including the bundle URI.</returns>
|
||||
Task<ProofBundle> CreateBundleAsync(
|
||||
SignedScanManifest signedManifest,
|
||||
ProofLedger ledger,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Read a proof bundle from storage.
|
||||
/// </summary>
|
||||
/// <param name="bundleUri">The URI to the bundle.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>The proof bundle contents.</returns>
|
||||
Task<ProofBundleContents> ReadBundleAsync(string bundleUri, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Metadata about a created proof bundle.
|
||||
/// </summary>
|
||||
/// <param name="ScanId">The scan ID this bundle belongs to.</param>
|
||||
/// <param name="RootHash">The root hash of the proof ledger.</param>
|
||||
/// <param name="BundleUri">URI where the bundle is stored.</param>
|
||||
/// <param name="CreatedAtUtc">When the bundle was created.</param>
|
||||
public sealed record ProofBundle(
|
||||
[property: JsonPropertyName("scanId")] string ScanId,
|
||||
[property: JsonPropertyName("rootHash")] string RootHash,
|
||||
[property: JsonPropertyName("bundleUri")] string BundleUri,
|
||||
[property: JsonPropertyName("createdAtUtc")] DateTimeOffset CreatedAtUtc);
|
||||
|
||||
/// <summary>
|
||||
/// Contents of a proof bundle when read from storage.
|
||||
/// </summary>
|
||||
/// <param name="Manifest">The scan manifest.</param>
|
||||
/// <param name="SignedManifest">The signed manifest with DSSE envelope.</param>
|
||||
/// <param name="ProofLedger">The proof ledger with all nodes.</param>
|
||||
/// <param name="Meta">Bundle metadata.</param>
|
||||
public sealed record ProofBundleContents(
|
||||
ScanManifest Manifest,
|
||||
SignedScanManifest SignedManifest,
|
||||
ProofLedger ProofLedger,
|
||||
ProofBundleMeta Meta);
|
||||
|
||||
/// <summary>
|
||||
/// Bundle metadata stored in meta.json.
|
||||
/// </summary>
|
||||
/// <param name="RootHash">Root hash of the proof ledger.</param>
|
||||
/// <param name="CreatedAtUtc">When the bundle was created.</param>
|
||||
/// <param name="Version">Bundle format version.</param>
|
||||
public sealed record ProofBundleMeta(
|
||||
[property: JsonPropertyName("rootHash")] string RootHash,
|
||||
[property: JsonPropertyName("createdAtUtc")] DateTimeOffset CreatedAtUtc,
|
||||
[property: JsonPropertyName("version")] string Version = "1.0");
|
||||
|
||||
/// <summary>
|
||||
/// Options for ProofBundleWriter.
|
||||
/// </summary>
|
||||
public sealed class ProofBundleWriterOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Base directory for storing proof bundles.
|
||||
/// </summary>
|
||||
public string StorageBasePath { get; set; } = "/var/lib/stellaops/proofs";
|
||||
|
||||
/// <summary>
|
||||
/// Whether to use content-addressed storage (bundle name = hash).
|
||||
/// </summary>
|
||||
public bool ContentAddressed { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Compression level for the ZIP bundle.
|
||||
/// </summary>
|
||||
public CompressionLevel CompressionLevel { get; set; } = CompressionLevel.Optimal;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of IProofBundleWriter.
|
||||
/// Creates ZIP bundles with the following structure:
|
||||
/// bundle.zip/
|
||||
/// ├── manifest.json # Canonical JSON scan manifest
|
||||
/// ├── manifest.dsse.json # DSSE envelope for manifest
|
||||
/// ├── score_proof.json # ProofLedger nodes array
|
||||
/// ├── proof_root.dsse.json # DSSE envelope for root hash (optional)
|
||||
/// └── meta.json # Bundle metadata
|
||||
/// </summary>
|
||||
public sealed class ProofBundleWriter : IProofBundleWriter
|
||||
{
|
||||
private readonly ProofBundleWriterOptions _options;
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
public ProofBundleWriter(ProofBundleWriterOptions? options = null)
|
||||
{
|
||||
_options = options ?? new ProofBundleWriterOptions();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ProofBundle> CreateBundleAsync(
|
||||
SignedScanManifest signedManifest,
|
||||
ProofLedger ledger,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(signedManifest);
|
||||
ArgumentNullException.ThrowIfNull(ledger);
|
||||
|
||||
var rootHash = ledger.RootHash();
|
||||
var createdAt = DateTimeOffset.UtcNow;
|
||||
|
||||
// Ensure storage directory exists
|
||||
Directory.CreateDirectory(_options.StorageBasePath);
|
||||
|
||||
// Determine bundle filename
|
||||
var bundleName = _options.ContentAddressed
|
||||
? $"{signedManifest.Manifest.ScanId}_{rootHash.Replace("sha256:", "")[..16]}.zip"
|
||||
: $"{signedManifest.Manifest.ScanId}.zip";
|
||||
|
||||
var bundlePath = Path.Combine(_options.StorageBasePath, bundleName);
|
||||
|
||||
// Create the ZIP bundle
|
||||
await CreateZipBundleAsync(bundlePath, signedManifest, ledger, rootHash, createdAt, cancellationToken);
|
||||
|
||||
return new ProofBundle(
|
||||
ScanId: signedManifest.Manifest.ScanId,
|
||||
RootHash: rootHash,
|
||||
BundleUri: bundlePath,
|
||||
CreatedAtUtc: createdAt);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ProofBundleContents> ReadBundleAsync(string bundleUri, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(bundleUri);
|
||||
|
||||
if (!File.Exists(bundleUri))
|
||||
throw new FileNotFoundException($"Proof bundle not found: {bundleUri}");
|
||||
|
||||
using var zipStream = new FileStream(bundleUri, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, useAsync: true);
|
||||
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read);
|
||||
|
||||
// Read manifest.json
|
||||
var manifestEntry = archive.GetEntry("manifest.json")
|
||||
?? throw new InvalidOperationException("Bundle missing manifest.json");
|
||||
var manifest = await ReadEntryAsAsync<ScanManifest>(manifestEntry, cancellationToken);
|
||||
|
||||
// Read manifest.dsse.json
|
||||
var signedManifestEntry = archive.GetEntry("manifest.dsse.json")
|
||||
?? throw new InvalidOperationException("Bundle missing manifest.dsse.json");
|
||||
var signedManifest = await ReadEntryAsAsync<SignedScanManifest>(signedManifestEntry, cancellationToken);
|
||||
|
||||
// Read score_proof.json
|
||||
var proofEntry = archive.GetEntry("score_proof.json")
|
||||
?? throw new InvalidOperationException("Bundle missing score_proof.json");
|
||||
var proofJson = await ReadEntryAsStringAsync(proofEntry, cancellationToken);
|
||||
var ledger = ProofLedger.FromJson(proofJson);
|
||||
|
||||
// Read meta.json
|
||||
var metaEntry = archive.GetEntry("meta.json")
|
||||
?? throw new InvalidOperationException("Bundle missing meta.json");
|
||||
var meta = await ReadEntryAsAsync<ProofBundleMeta>(metaEntry, cancellationToken);
|
||||
|
||||
return new ProofBundleContents(manifest, signedManifest, ledger, meta);
|
||||
}
|
||||
|
||||
private async Task CreateZipBundleAsync(
|
||||
string bundlePath,
|
||||
SignedScanManifest signedManifest,
|
||||
ProofLedger ledger,
|
||||
string rootHash,
|
||||
DateTimeOffset createdAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Write to a temp file first, then move (atomic on most filesystems)
|
||||
var tempPath = bundlePath + ".tmp";
|
||||
|
||||
try
|
||||
{
|
||||
await using (var zipStream = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, useAsync: true))
|
||||
using (var archive = new ZipArchive(zipStream, ZipArchiveMode.Create))
|
||||
{
|
||||
// manifest.json - canonical manifest
|
||||
await WriteEntryAsync(archive, "manifest.json", signedManifest.Manifest.ToJson(indented: true), cancellationToken);
|
||||
|
||||
// manifest.dsse.json - signed manifest with envelope
|
||||
await WriteEntryAsync(archive, "manifest.dsse.json", signedManifest.ToJson(indented: true), cancellationToken);
|
||||
|
||||
// score_proof.json - proof ledger
|
||||
await WriteEntryAsync(archive, "score_proof.json", ledger.ToJson(JsonOptions), cancellationToken);
|
||||
|
||||
// meta.json - bundle metadata
|
||||
var meta = new ProofBundleMeta(rootHash, createdAt);
|
||||
await WriteEntryAsync(archive, "meta.json", JsonSerializer.Serialize(meta, JsonOptions), cancellationToken);
|
||||
}
|
||||
|
||||
// Atomic move
|
||||
File.Move(tempPath, bundlePath, overwrite: true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Clean up temp file if it still exists
|
||||
if (File.Exists(tempPath))
|
||||
File.Delete(tempPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WriteEntryAsync(ZipArchive archive, string entryName, string content, CancellationToken cancellationToken)
|
||||
{
|
||||
var entry = archive.CreateEntry(entryName, CompressionLevel.Optimal);
|
||||
await using var entryStream = entry.Open();
|
||||
var bytes = Encoding.UTF8.GetBytes(content);
|
||||
await entryStream.WriteAsync(bytes, cancellationToken);
|
||||
}
|
||||
|
||||
private static async Task<T> ReadEntryAsAsync<T>(ZipArchiveEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var entryStream = entry.Open();
|
||||
return await JsonSerializer.DeserializeAsync<T>(entryStream, JsonOptions, cancellationToken)
|
||||
?? throw new InvalidOperationException($"Failed to deserialize {entry.FullName}");
|
||||
}
|
||||
|
||||
private static async Task<string> ReadEntryAsStringAsync(ZipArchiveEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var entryStream = entry.Open();
|
||||
using var reader = new StreamReader(entryStream, Encoding.UTF8);
|
||||
return await reader.ReadToEndAsync(cancellationToken);
|
||||
}
|
||||
}
|
||||
201
src/Scanner/__Libraries/StellaOps.Scanner.Core/ScanManifest.cs
Normal file
201
src/Scanner/__Libraries/StellaOps.Scanner.Core/ScanManifest.cs
Normal file
@@ -0,0 +1,201 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ScanManifest.cs
|
||||
// Sprint: SPRINT_3401_0002_0001_score_replay_proof_bundle
|
||||
// Task: SCORE-REPLAY-005 - Define ScanManifest record with all input hashes
|
||||
// Description: Captures all inputs affecting scan results for reproducibility
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.Core;
|
||||
|
||||
/// <summary>
|
||||
/// Captures all inputs that affect a scan's results.
|
||||
/// Per advisory "Building a Deeper Moat Beyond Reachability" §12.
|
||||
/// This manifest ensures reproducibility: same manifest + same seed = same results.
|
||||
/// </summary>
|
||||
/// <param name="ScanId">Unique identifier for this scan run.</param>
|
||||
/// <param name="CreatedAtUtc">When the scan was initiated (UTC).</param>
|
||||
/// <param name="ArtifactDigest">SHA-256 digest of the scanned artifact (e.g., "sha256:abc...").</param>
|
||||
/// <param name="ArtifactPurl">Optional Package URL for the artifact.</param>
|
||||
/// <param name="ScannerVersion">Version of the scanner webservice.</param>
|
||||
/// <param name="WorkerVersion">Version of the scanner worker that performed the scan.</param>
|
||||
/// <param name="ConcelierSnapshotHash">Digest of the immutable feed snapshot from Concelier.</param>
|
||||
/// <param name="ExcititorSnapshotHash">Digest of the immutable VEX snapshot from Excititor.</param>
|
||||
/// <param name="LatticePolicyHash">Digest of the policy bundle used for evaluation.</param>
|
||||
/// <param name="Deterministic">Whether the scan was run in deterministic mode.</param>
|
||||
/// <param name="Seed">32-byte seed for deterministic replay.</param>
|
||||
/// <param name="Knobs">Configuration knobs affecting the scan (depth limits, etc.).</param>
|
||||
public sealed record ScanManifest(
|
||||
[property: JsonPropertyName("scanId")] string ScanId,
|
||||
[property: JsonPropertyName("createdAtUtc")] DateTimeOffset CreatedAtUtc,
|
||||
[property: JsonPropertyName("artifactDigest")] string ArtifactDigest,
|
||||
[property: JsonPropertyName("artifactPurl")] string? ArtifactPurl,
|
||||
[property: JsonPropertyName("scannerVersion")] string ScannerVersion,
|
||||
[property: JsonPropertyName("workerVersion")] string WorkerVersion,
|
||||
[property: JsonPropertyName("concelierSnapshotHash")] string ConcelierSnapshotHash,
|
||||
[property: JsonPropertyName("excititorSnapshotHash")] string ExcititorSnapshotHash,
|
||||
[property: JsonPropertyName("latticePolicyHash")] string LatticePolicyHash,
|
||||
[property: JsonPropertyName("deterministic")] bool Deterministic,
|
||||
[property: JsonPropertyName("seed")] byte[] Seed,
|
||||
[property: JsonPropertyName("knobs")] IReadOnlyDictionary<string, string> Knobs)
|
||||
{
|
||||
/// <summary>
|
||||
/// Default JSON serializer options for canonical output.
|
||||
/// </summary>
|
||||
private static readonly JsonSerializerOptions CanonicalJsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
WriteIndented = false,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Create a manifest builder with required fields.
|
||||
/// </summary>
|
||||
public static ScanManifestBuilder CreateBuilder(string scanId, string artifactDigest) =>
|
||||
new(scanId, artifactDigest);
|
||||
|
||||
/// <summary>
|
||||
/// Serialize to canonical JSON (for hashing).
|
||||
/// </summary>
|
||||
public string ToCanonicalJson() => JsonSerializer.Serialize(this, CanonicalJsonOptions);
|
||||
|
||||
/// <summary>
|
||||
/// Compute the SHA-256 hash of the canonical JSON representation.
|
||||
/// </summary>
|
||||
public string ComputeHash()
|
||||
{
|
||||
var json = ToCanonicalJson();
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes(json);
|
||||
var hash = System.Security.Cryptography.SHA256.HashData(bytes);
|
||||
return $"sha256:{Convert.ToHexStringLower(hash)}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deserialize from JSON.
|
||||
/// </summary>
|
||||
public static ScanManifest FromJson(string json) =>
|
||||
JsonSerializer.Deserialize<ScanManifest>(json, CanonicalJsonOptions)
|
||||
?? throw new InvalidOperationException("Failed to deserialize ScanManifest");
|
||||
|
||||
/// <summary>
|
||||
/// Serialize to JSON.
|
||||
/// </summary>
|
||||
public string ToJson(bool indented = false)
|
||||
{
|
||||
var options = indented
|
||||
? new JsonSerializerOptions(CanonicalJsonOptions) { WriteIndented = true }
|
||||
: CanonicalJsonOptions;
|
||||
return JsonSerializer.Serialize(this, options);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builder for creating ScanManifest instances.
|
||||
/// </summary>
|
||||
public sealed class ScanManifestBuilder
|
||||
{
|
||||
private readonly string _scanId;
|
||||
private readonly string _artifactDigest;
|
||||
private DateTimeOffset _createdAtUtc = DateTimeOffset.UtcNow;
|
||||
private string? _artifactPurl;
|
||||
private string _scannerVersion = "1.0.0";
|
||||
private string _workerVersion = "1.0.0";
|
||||
private string _concelierSnapshotHash = string.Empty;
|
||||
private string _excititorSnapshotHash = string.Empty;
|
||||
private string _latticePolicyHash = string.Empty;
|
||||
private bool _deterministic = true;
|
||||
private byte[] _seed = new byte[32];
|
||||
private readonly Dictionary<string, string> _knobs = [];
|
||||
|
||||
internal ScanManifestBuilder(string scanId, string artifactDigest)
|
||||
{
|
||||
_scanId = scanId ?? throw new ArgumentNullException(nameof(scanId));
|
||||
_artifactDigest = artifactDigest ?? throw new ArgumentNullException(nameof(artifactDigest));
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithCreatedAt(DateTimeOffset createdAtUtc)
|
||||
{
|
||||
_createdAtUtc = createdAtUtc;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithArtifactPurl(string purl)
|
||||
{
|
||||
_artifactPurl = purl;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithScannerVersion(string version)
|
||||
{
|
||||
_scannerVersion = version;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithWorkerVersion(string version)
|
||||
{
|
||||
_workerVersion = version;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithConcelierSnapshot(string hash)
|
||||
{
|
||||
_concelierSnapshotHash = hash;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithExcititorSnapshot(string hash)
|
||||
{
|
||||
_excititorSnapshotHash = hash;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithLatticePolicyHash(string hash)
|
||||
{
|
||||
_latticePolicyHash = hash;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithDeterministic(bool deterministic)
|
||||
{
|
||||
_deterministic = deterministic;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithSeed(byte[] seed)
|
||||
{
|
||||
if (seed.Length != 32)
|
||||
throw new ArgumentException("Seed must be 32 bytes", nameof(seed));
|
||||
_seed = seed;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithKnob(string key, string value)
|
||||
{
|
||||
_knobs[key] = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifestBuilder WithKnobs(IReadOnlyDictionary<string, string> knobs)
|
||||
{
|
||||
foreach (var (key, value) in knobs)
|
||||
_knobs[key] = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
public ScanManifest Build() => new(
|
||||
ScanId: _scanId,
|
||||
CreatedAtUtc: _createdAtUtc,
|
||||
ArtifactDigest: _artifactDigest,
|
||||
ArtifactPurl: _artifactPurl,
|
||||
ScannerVersion: _scannerVersion,
|
||||
WorkerVersion: _workerVersion,
|
||||
ConcelierSnapshotHash: _concelierSnapshotHash,
|
||||
ExcititorSnapshotHash: _excititorSnapshotHash,
|
||||
LatticePolicyHash: _latticePolicyHash,
|
||||
Deterministic: _deterministic,
|
||||
Seed: _seed,
|
||||
Knobs: _knobs.AsReadOnly());
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ScanManifestSigner.cs
|
||||
// Sprint: SPRINT_3401_0002_0001_score_replay_proof_bundle
|
||||
// Task: SCORE-REPLAY-006 - Implement manifest DSSE signing
|
||||
// Description: Signs scan manifests using DSSE envelope format
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Scanner.ProofSpine;
|
||||
|
||||
namespace StellaOps.Scanner.Core;
|
||||
|
||||
/// <summary>
|
||||
/// Service for signing scan manifests using DSSE format.
|
||||
/// </summary>
|
||||
public interface IScanManifestSigner
|
||||
{
|
||||
/// <summary>
|
||||
/// Sign a scan manifest and produce a DSSE envelope.
|
||||
/// </summary>
|
||||
/// <param name="manifest">The manifest to sign.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A signed DSSE envelope containing the manifest.</returns>
|
||||
Task<SignedScanManifest> SignAsync(ScanManifest manifest, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Verify a signed manifest envelope.
|
||||
/// </summary>
|
||||
/// <param name="signedManifest">The signed manifest to verify.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>Verification result with the extracted manifest if valid.</returns>
|
||||
Task<ManifestVerificationResult> VerifyAsync(SignedScanManifest signedManifest, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A signed scan manifest with DSSE envelope.
|
||||
/// </summary>
|
||||
/// <param name="Manifest">The original scan manifest.</param>
|
||||
/// <param name="ManifestHash">SHA-256 hash of the canonical manifest JSON.</param>
|
||||
/// <param name="Envelope">The DSSE envelope containing the signed manifest.</param>
|
||||
/// <param name="SignedAt">When the manifest was signed (UTC).</param>
|
||||
public sealed record SignedScanManifest(
|
||||
[property: JsonPropertyName("manifest")] ScanManifest Manifest,
|
||||
[property: JsonPropertyName("manifestHash")] string ManifestHash,
|
||||
[property: JsonPropertyName("envelope")] DsseEnvelope Envelope,
|
||||
[property: JsonPropertyName("signedAt")] DateTimeOffset SignedAt)
|
||||
{
|
||||
/// <summary>
|
||||
/// Serialize to JSON.
|
||||
/// </summary>
|
||||
public string ToJson(bool indented = false) =>
|
||||
JsonSerializer.Serialize(this, new JsonSerializerOptions { WriteIndented = indented });
|
||||
|
||||
/// <summary>
|
||||
/// Deserialize from JSON.
|
||||
/// </summary>
|
||||
public static SignedScanManifest FromJson(string json) =>
|
||||
JsonSerializer.Deserialize<SignedScanManifest>(json)
|
||||
?? throw new InvalidOperationException("Failed to deserialize SignedScanManifest");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of manifest verification.
|
||||
/// </summary>
|
||||
/// <param name="IsValid">Whether the signature is valid.</param>
|
||||
/// <param name="Manifest">The extracted manifest if valid, null otherwise.</param>
|
||||
/// <param name="VerifiedAt">When verification was performed.</param>
|
||||
/// <param name="ErrorMessage">Error message if verification failed.</param>
|
||||
/// <param name="KeyId">The key ID that was used for signing.</param>
|
||||
public sealed record ManifestVerificationResult(
|
||||
bool IsValid,
|
||||
ScanManifest? Manifest,
|
||||
DateTimeOffset VerifiedAt,
|
||||
string? ErrorMessage = null,
|
||||
string? KeyId = null)
|
||||
{
|
||||
public static ManifestVerificationResult Success(ScanManifest manifest, string? keyId = null) =>
|
||||
new(true, manifest, DateTimeOffset.UtcNow, null, keyId);
|
||||
|
||||
public static ManifestVerificationResult Failure(string error) =>
|
||||
new(false, null, DateTimeOffset.UtcNow, error);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of IScanManifestSigner using DSSE.
|
||||
/// </summary>
|
||||
public sealed class ScanManifestSigner : IScanManifestSigner
|
||||
{
|
||||
private readonly IDsseSigningService _dsseSigningService;
|
||||
private const string PredicateType = "scanmanifest.stella/v1";
|
||||
|
||||
public ScanManifestSigner(IDsseSigningService dsseSigningService)
|
||||
{
|
||||
_dsseSigningService = dsseSigningService ?? throw new ArgumentNullException(nameof(dsseSigningService));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<SignedScanManifest> SignAsync(ScanManifest manifest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
|
||||
var manifestHash = manifest.ComputeHash();
|
||||
var manifestJson = manifest.ToCanonicalJson();
|
||||
var manifestBytes = System.Text.Encoding.UTF8.GetBytes(manifestJson);
|
||||
|
||||
// Create DSSE envelope
|
||||
var envelope = await _dsseSigningService.SignAsync(
|
||||
payloadType: PredicateType,
|
||||
payload: manifestBytes,
|
||||
cancellationToken);
|
||||
|
||||
return new SignedScanManifest(
|
||||
Manifest: manifest,
|
||||
ManifestHash: manifestHash,
|
||||
Envelope: envelope,
|
||||
SignedAt: DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<ManifestVerificationResult> VerifyAsync(SignedScanManifest signedManifest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(signedManifest);
|
||||
|
||||
try
|
||||
{
|
||||
// Verify DSSE signature
|
||||
var verifyResult = await _dsseSigningService.VerifyAsync(signedManifest.Envelope, cancellationToken);
|
||||
if (!verifyResult)
|
||||
{
|
||||
return ManifestVerificationResult.Failure("DSSE signature verification failed");
|
||||
}
|
||||
|
||||
// Verify payload type
|
||||
if (signedManifest.Envelope.PayloadType != PredicateType)
|
||||
{
|
||||
return ManifestVerificationResult.Failure($"Unexpected payload type: {signedManifest.Envelope.PayloadType}");
|
||||
}
|
||||
|
||||
// Verify manifest hash
|
||||
var computedHash = signedManifest.Manifest.ComputeHash();
|
||||
if (computedHash != signedManifest.ManifestHash)
|
||||
{
|
||||
return ManifestVerificationResult.Failure("Manifest hash mismatch");
|
||||
}
|
||||
|
||||
var keyId = signedManifest.Envelope.Signatures.FirstOrDefault()?.Keyid;
|
||||
return ManifestVerificationResult.Success(signedManifest.Manifest, keyId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return ManifestVerificationResult.Failure($"Verification error: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// SmartDiffScoringConfig.cs
|
||||
// Sprint: SPRINT_3500_0004_0001_smart_diff_binary_output
|
||||
// Task: SDIFF-BIN-019 - Implement SmartDiffScoringConfig with presets
|
||||
// Task: SDIFF-BIN-021 - Implement ToDetectorOptions() conversion
|
||||
// Description: Configurable scoring weights for Smart-Diff detection
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.SmartDiff.Detection;
|
||||
|
||||
/// <summary>
|
||||
/// Comprehensive configuration for Smart-Diff scoring.
|
||||
/// Exposes all configurable weights and thresholds for risk detection.
|
||||
/// Per Sprint 3500.4 - Smart-Diff Scoring Configuration.
|
||||
/// </summary>
|
||||
public sealed class SmartDiffScoringConfig
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration name/identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("name")]
|
||||
public string Name { get; init; } = "default";
|
||||
|
||||
/// <summary>
|
||||
/// Configuration version for compatibility tracking.
|
||||
/// </summary>
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; init; } = "1.0";
|
||||
|
||||
#region Rule R1: Reachability
|
||||
|
||||
/// <summary>
|
||||
/// Weight for reachability flip from unreachable to reachable (risk increase).
|
||||
/// </summary>
|
||||
[JsonPropertyName("reachabilityFlipUpWeight")]
|
||||
public double ReachabilityFlipUpWeight { get; init; } = 1.0;
|
||||
|
||||
/// <summary>
|
||||
/// Weight for reachability flip from reachable to unreachable (risk decrease).
|
||||
/// </summary>
|
||||
[JsonPropertyName("reachabilityFlipDownWeight")]
|
||||
public double ReachabilityFlipDownWeight { get; init; } = 0.8;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to consider lattice confidence in reachability scoring.
|
||||
/// </summary>
|
||||
[JsonPropertyName("useLatticeConfidence")]
|
||||
public bool UseLatticeConfidence { get; init; } = true;
|
||||
|
||||
#endregion
|
||||
|
||||
#region Rule R2: VEX Status
|
||||
|
||||
/// <summary>
|
||||
/// Weight for VEX status flip to affected.
|
||||
/// </summary>
|
||||
[JsonPropertyName("vexFlipToAffectedWeight")]
|
||||
public double VexFlipToAffectedWeight { get; init; } = 0.9;
|
||||
|
||||
/// <summary>
|
||||
/// Weight for VEX status flip to not_affected.
|
||||
/// </summary>
|
||||
[JsonPropertyName("vexFlipToNotAffectedWeight")]
|
||||
public double VexFlipToNotAffectedWeight { get; init; } = 0.7;
|
||||
|
||||
/// <summary>
|
||||
/// Weight for VEX status flip to fixed.
|
||||
/// </summary>
|
||||
[JsonPropertyName("vexFlipToFixedWeight")]
|
||||
public double VexFlipToFixedWeight { get; init; } = 0.6;
|
||||
|
||||
/// <summary>
|
||||
/// Weight for VEX status flip to under_investigation.
|
||||
/// </summary>
|
||||
[JsonPropertyName("vexFlipToUnderInvestigationWeight")]
|
||||
public double VexFlipToUnderInvestigationWeight { get; init; } = 0.3;
|
||||
|
||||
#endregion
|
||||
|
||||
#region Rule R3: Affected Range
|
||||
|
||||
/// <summary>
|
||||
/// Weight for entering the affected version range.
|
||||
/// </summary>
|
||||
[JsonPropertyName("rangeEntryWeight")]
|
||||
public double RangeEntryWeight { get; init; } = 0.8;
|
||||
|
||||
/// <summary>
|
||||
/// Weight for exiting the affected version range.
|
||||
/// </summary>
|
||||
[JsonPropertyName("rangeExitWeight")]
|
||||
public double RangeExitWeight { get; init; } = 0.6;
|
||||
|
||||
#endregion
|
||||
|
||||
#region Rule R4: Intelligence Signals
|
||||
|
||||
/// <summary>
|
||||
/// Weight for KEV (Known Exploited Vulnerability) addition.
|
||||
/// </summary>
|
||||
[JsonPropertyName("kevAddedWeight")]
|
||||
public double KevAddedWeight { get; init; } = 1.0;
|
||||
|
||||
/// <summary>
|
||||
/// Weight for KEV removal.
|
||||
/// </summary>
|
||||
[JsonPropertyName("kevRemovedWeight")]
|
||||
public double KevRemovedWeight { get; init; } = 0.5;
|
||||
|
||||
/// <summary>
|
||||
/// Weight for EPSS threshold crossing.
|
||||
/// </summary>
|
||||
[JsonPropertyName("epssThresholdWeight")]
|
||||
public double EpssThresholdWeight { get; init; } = 0.6;
|
||||
|
||||
/// <summary>
|
||||
/// EPSS score threshold for R4 detection (0.0 - 1.0).
|
||||
/// </summary>
|
||||
[JsonPropertyName("epssThreshold")]
|
||||
public double EpssThreshold { get; init; } = 0.5;
|
||||
|
||||
/// <summary>
|
||||
/// Weight for policy decision flip.
|
||||
/// </summary>
|
||||
[JsonPropertyName("policyFlipWeight")]
|
||||
public double PolicyFlipWeight { get; init; } = 0.7;
|
||||
|
||||
#endregion
|
||||
|
||||
#region Hardening Detection
|
||||
|
||||
/// <summary>
|
||||
/// Weight for hardening regression detection.
|
||||
/// </summary>
|
||||
[JsonPropertyName("hardeningRegressionWeight")]
|
||||
public double HardeningRegressionWeight { get; init; } = 0.7;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum hardening score difference to trigger a finding.
|
||||
/// </summary>
|
||||
[JsonPropertyName("hardeningScoreThreshold")]
|
||||
public double HardeningScoreThreshold { get; init; } = 0.2;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to include hardening flags in diff output.
|
||||
/// </summary>
|
||||
[JsonPropertyName("includeHardeningFlags")]
|
||||
public bool IncludeHardeningFlags { get; init; } = true;
|
||||
|
||||
#endregion
|
||||
|
||||
#region Priority Score Factors
|
||||
|
||||
/// <summary>
|
||||
/// Multiplier applied when finding is in KEV.
|
||||
/// </summary>
|
||||
[JsonPropertyName("kevBoost")]
|
||||
public double KevBoost { get; init; } = 1.5;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum priority score to emit a finding.
|
||||
/// </summary>
|
||||
[JsonPropertyName("minPriorityScore")]
|
||||
public double MinPriorityScore { get; init; } = 0.1;
|
||||
|
||||
/// <summary>
|
||||
/// Threshold for "high priority" classification.
|
||||
/// </summary>
|
||||
[JsonPropertyName("highPriorityThreshold")]
|
||||
public double HighPriorityThreshold { get; init; } = 0.7;
|
||||
|
||||
/// <summary>
|
||||
/// Threshold for "critical priority" classification.
|
||||
/// </summary>
|
||||
[JsonPropertyName("criticalPriorityThreshold")]
|
||||
public double CriticalPriorityThreshold { get; init; } = 0.9;
|
||||
|
||||
#endregion
|
||||
|
||||
#region Presets
|
||||
|
||||
/// <summary>
|
||||
/// Default configuration - balanced detection.
|
||||
/// </summary>
|
||||
public static SmartDiffScoringConfig Default => new()
|
||||
{
|
||||
Name = "default"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Security-focused preset - aggressive detection, lower thresholds.
|
||||
/// </summary>
|
||||
public static SmartDiffScoringConfig SecurityFocused => new()
|
||||
{
|
||||
Name = "security-focused",
|
||||
ReachabilityFlipUpWeight = 1.2,
|
||||
VexFlipToAffectedWeight = 1.0,
|
||||
KevAddedWeight = 1.5,
|
||||
EpssThreshold = 0.3,
|
||||
EpssThresholdWeight = 0.8,
|
||||
HardeningRegressionWeight = 0.9,
|
||||
HardeningScoreThreshold = 0.15,
|
||||
MinPriorityScore = 0.05,
|
||||
HighPriorityThreshold = 0.5,
|
||||
CriticalPriorityThreshold = 0.8
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Compliance-focused preset - stricter thresholds for regulated environments.
|
||||
/// </summary>
|
||||
public static SmartDiffScoringConfig ComplianceFocused => new()
|
||||
{
|
||||
Name = "compliance-focused",
|
||||
ReachabilityFlipUpWeight = 1.0,
|
||||
VexFlipToAffectedWeight = 1.0,
|
||||
VexFlipToNotAffectedWeight = 0.9,
|
||||
KevAddedWeight = 2.0,
|
||||
EpssThreshold = 0.2,
|
||||
PolicyFlipWeight = 1.0,
|
||||
HardeningRegressionWeight = 1.0,
|
||||
HardeningScoreThreshold = 0.1,
|
||||
MinPriorityScore = 0.0,
|
||||
HighPriorityThreshold = 0.4,
|
||||
CriticalPriorityThreshold = 0.7
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Developer-friendly preset - reduced noise, focus on actionable changes.
|
||||
/// </summary>
|
||||
public static SmartDiffScoringConfig DeveloperFriendly => new()
|
||||
{
|
||||
Name = "developer-friendly",
|
||||
ReachabilityFlipUpWeight = 0.8,
|
||||
VexFlipToAffectedWeight = 0.7,
|
||||
KevAddedWeight = 1.0,
|
||||
EpssThreshold = 0.7,
|
||||
EpssThresholdWeight = 0.4,
|
||||
HardeningRegressionWeight = 0.5,
|
||||
HardeningScoreThreshold = 0.3,
|
||||
MinPriorityScore = 0.2,
|
||||
HighPriorityThreshold = 0.8,
|
||||
CriticalPriorityThreshold = 0.95
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Get a preset configuration by name.
|
||||
/// </summary>
|
||||
public static SmartDiffScoringConfig GetPreset(string name) => name.ToLowerInvariant() switch
|
||||
{
|
||||
"default" => Default,
|
||||
"security-focused" or "security" => SecurityFocused,
|
||||
"compliance-focused" or "compliance" => ComplianceFocused,
|
||||
"developer-friendly" or "developer" => DeveloperFriendly,
|
||||
_ => throw new ArgumentException($"Unknown scoring preset: {name}")
|
||||
};
|
||||
|
||||
#endregion
|
||||
|
||||
#region Conversion Methods
|
||||
|
||||
/// <summary>
|
||||
/// Convert to MaterialRiskChangeOptions for use with the detector.
|
||||
/// Task: SDIFF-BIN-021.
|
||||
/// </summary>
|
||||
public MaterialRiskChangeOptions ToDetectorOptions() => new()
|
||||
{
|
||||
ReachabilityFlipUpWeight = ReachabilityFlipUpWeight,
|
||||
ReachabilityFlipDownWeight = ReachabilityFlipDownWeight,
|
||||
VexFlipToAffectedWeight = VexFlipToAffectedWeight,
|
||||
VexFlipToNotAffectedWeight = VexFlipToNotAffectedWeight,
|
||||
RangeEntryWeight = RangeEntryWeight,
|
||||
RangeExitWeight = RangeExitWeight,
|
||||
KevAddedWeight = KevAddedWeight,
|
||||
KevRemovedWeight = KevRemovedWeight,
|
||||
EpssThreshold = EpssThreshold,
|
||||
EpssThresholdWeight = EpssThresholdWeight,
|
||||
PolicyFlipWeight = PolicyFlipWeight
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Create a detector configured with these options.
|
||||
/// </summary>
|
||||
public MaterialRiskChangeDetector CreateDetector() => new(ToDetectorOptions());
|
||||
|
||||
/// <summary>
|
||||
/// Validate configuration values.
|
||||
/// </summary>
|
||||
public SmartDiffScoringConfigValidation Validate()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
// Weight validations (should be 0.0 - 2.0)
|
||||
ValidateWeight(nameof(ReachabilityFlipUpWeight), ReachabilityFlipUpWeight, errors);
|
||||
ValidateWeight(nameof(ReachabilityFlipDownWeight), ReachabilityFlipDownWeight, errors);
|
||||
ValidateWeight(nameof(VexFlipToAffectedWeight), VexFlipToAffectedWeight, errors);
|
||||
ValidateWeight(nameof(VexFlipToNotAffectedWeight), VexFlipToNotAffectedWeight, errors);
|
||||
ValidateWeight(nameof(RangeEntryWeight), RangeEntryWeight, errors);
|
||||
ValidateWeight(nameof(RangeExitWeight), RangeExitWeight, errors);
|
||||
ValidateWeight(nameof(KevAddedWeight), KevAddedWeight, errors);
|
||||
ValidateWeight(nameof(KevRemovedWeight), KevRemovedWeight, errors);
|
||||
ValidateWeight(nameof(EpssThresholdWeight), EpssThresholdWeight, errors);
|
||||
ValidateWeight(nameof(PolicyFlipWeight), PolicyFlipWeight, errors);
|
||||
ValidateWeight(nameof(HardeningRegressionWeight), HardeningRegressionWeight, errors);
|
||||
|
||||
// Threshold validations (should be 0.0 - 1.0)
|
||||
ValidateThreshold(nameof(EpssThreshold), EpssThreshold, errors);
|
||||
ValidateThreshold(nameof(HardeningScoreThreshold), HardeningScoreThreshold, errors);
|
||||
ValidateThreshold(nameof(MinPriorityScore), MinPriorityScore, errors);
|
||||
ValidateThreshold(nameof(HighPriorityThreshold), HighPriorityThreshold, errors);
|
||||
ValidateThreshold(nameof(CriticalPriorityThreshold), CriticalPriorityThreshold, errors);
|
||||
|
||||
// Logical validations
|
||||
if (HighPriorityThreshold >= CriticalPriorityThreshold)
|
||||
{
|
||||
errors.Add($"HighPriorityThreshold ({HighPriorityThreshold}) must be less than CriticalPriorityThreshold ({CriticalPriorityThreshold})");
|
||||
}
|
||||
|
||||
if (MinPriorityScore >= HighPriorityThreshold)
|
||||
{
|
||||
errors.Add($"MinPriorityScore ({MinPriorityScore}) should be less than HighPriorityThreshold ({HighPriorityThreshold})");
|
||||
}
|
||||
|
||||
return new SmartDiffScoringConfigValidation(errors.Count == 0, [.. errors]);
|
||||
}
|
||||
|
||||
private static void ValidateWeight(string name, double value, List<string> errors)
|
||||
{
|
||||
if (value < 0.0 || value > 2.0)
|
||||
{
|
||||
errors.Add($"{name} must be between 0.0 and 2.0, got {value}");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateThreshold(string name, double value, List<string> errors)
|
||||
{
|
||||
if (value < 0.0 || value > 1.0)
|
||||
{
|
||||
errors.Add($"{name} must be between 0.0 and 1.0, got {value}");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of scoring config validation.
|
||||
/// </summary>
|
||||
public sealed record SmartDiffScoringConfigValidation(
|
||||
[property: JsonPropertyName("isValid")] bool IsValid,
|
||||
[property: JsonPropertyName("errors")] string[] Errors);
|
||||
@@ -0,0 +1,117 @@
|
||||
-- Migration: 006_score_replay_tables.sql
|
||||
-- Sprint: SPRINT_3401_0002_0001
|
||||
-- Tasks: SCORE-REPLAY-007 (scan_manifest), SCORE-REPLAY-009 (proof_bundle)
|
||||
-- Description: Tables for score replay and proof bundle functionality
|
||||
|
||||
-- Scan manifests for deterministic replay
|
||||
CREATE TABLE IF NOT EXISTS scan_manifest (
|
||||
manifest_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
scan_id UUID NOT NULL,
|
||||
manifest_hash VARCHAR(128) NOT NULL, -- SHA-256 of manifest content
|
||||
sbom_hash VARCHAR(128) NOT NULL, -- Hash of input SBOM
|
||||
rules_hash VARCHAR(128) NOT NULL, -- Hash of rules snapshot
|
||||
feed_hash VARCHAR(128) NOT NULL, -- Hash of advisory feed snapshot
|
||||
policy_hash VARCHAR(128) NOT NULL, -- Hash of scoring policy
|
||||
|
||||
-- Evidence timing
|
||||
scan_started_at TIMESTAMPTZ NOT NULL,
|
||||
scan_completed_at TIMESTAMPTZ,
|
||||
|
||||
-- Content (stored as JSONB for query flexibility)
|
||||
manifest_content JSONB NOT NULL,
|
||||
|
||||
-- Metadata
|
||||
scanner_version VARCHAR(64) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
-- Constraints
|
||||
CONSTRAINT fk_scan_manifest_scan FOREIGN KEY (scan_id) REFERENCES scans(scan_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for manifest hash lookups (for deduplication and verification)
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_manifest_hash ON scan_manifest(manifest_hash);
|
||||
|
||||
-- Index for scan lookups
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_manifest_scan_id ON scan_manifest(scan_id);
|
||||
|
||||
-- Index for temporal queries
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_manifest_created_at ON scan_manifest(created_at DESC);
|
||||
|
||||
-- Proof bundles for cryptographic evidence chains
|
||||
CREATE TABLE IF NOT EXISTS proof_bundle (
|
||||
scan_id UUID NOT NULL,
|
||||
root_hash VARCHAR(128) NOT NULL, -- Merkle root of all evidence
|
||||
bundle_type VARCHAR(32) NOT NULL DEFAULT 'standard', -- 'standard', 'extended', 'minimal'
|
||||
|
||||
-- DSSE envelope for the bundle
|
||||
dsse_envelope JSONB, -- Full DSSE-signed envelope
|
||||
signature_keyid VARCHAR(256), -- Key ID used for signing
|
||||
signature_algorithm VARCHAR(64), -- e.g., 'ed25519', 'rsa-pss-sha256'
|
||||
|
||||
-- Bundle content
|
||||
bundle_content BYTEA, -- ZIP archive or raw bundle data
|
||||
bundle_hash VARCHAR(128) NOT NULL, -- SHA-256 of bundle_content
|
||||
|
||||
-- Component hashes for incremental verification
|
||||
ledger_hash VARCHAR(128), -- Hash of proof ledger
|
||||
manifest_hash VARCHAR(128), -- Reference to scan_manifest
|
||||
sbom_hash VARCHAR(128),
|
||||
vex_hash VARCHAR(128),
|
||||
|
||||
-- Metadata
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
expires_at TIMESTAMPTZ, -- Optional TTL for retention
|
||||
|
||||
-- Primary key is (scan_id, root_hash) to allow multiple bundles per scan
|
||||
PRIMARY KEY (scan_id, root_hash),
|
||||
|
||||
-- Foreign key
|
||||
CONSTRAINT fk_proof_bundle_scan FOREIGN KEY (scan_id) REFERENCES scans(scan_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for root hash lookups (for verification)
|
||||
CREATE INDEX IF NOT EXISTS idx_proof_bundle_root_hash ON proof_bundle(root_hash);
|
||||
|
||||
-- Index for temporal queries
|
||||
CREATE INDEX IF NOT EXISTS idx_proof_bundle_created_at ON proof_bundle(created_at DESC);
|
||||
|
||||
-- Index for expiration cleanup
|
||||
CREATE INDEX IF NOT EXISTS idx_proof_bundle_expires_at ON proof_bundle(expires_at) WHERE expires_at IS NOT NULL;
|
||||
|
||||
-- Score replay history for tracking rescores
|
||||
CREATE TABLE IF NOT EXISTS score_replay_history (
|
||||
replay_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
scan_id UUID NOT NULL,
|
||||
|
||||
-- What triggered the replay
|
||||
trigger_type VARCHAR(32) NOT NULL, -- 'feed_update', 'policy_change', 'manual', 'scheduled'
|
||||
trigger_reference VARCHAR(256), -- Feed snapshot ID, policy version, etc.
|
||||
|
||||
-- Before/after state
|
||||
original_manifest_hash VARCHAR(128),
|
||||
replayed_manifest_hash VARCHAR(128),
|
||||
|
||||
-- Score delta summary
|
||||
score_delta_json JSONB, -- Summary of changed scores
|
||||
findings_added INT DEFAULT 0,
|
||||
findings_removed INT DEFAULT 0,
|
||||
findings_rescored INT DEFAULT 0,
|
||||
|
||||
-- Timing
|
||||
replayed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
duration_ms INT,
|
||||
|
||||
-- Foreign key
|
||||
CONSTRAINT fk_score_replay_scan FOREIGN KEY (scan_id) REFERENCES scans(scan_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for scan-based lookups
|
||||
CREATE INDEX IF NOT EXISTS idx_score_replay_scan_id ON score_replay_history(scan_id);
|
||||
|
||||
-- Index for temporal queries
|
||||
CREATE INDEX IF NOT EXISTS idx_score_replay_replayed_at ON score_replay_history(replayed_at DESC);
|
||||
|
||||
-- Comments for documentation
|
||||
COMMENT ON TABLE scan_manifest IS 'Deterministic scan manifests for score replay. Each manifest captures all inputs needed to reproduce a scan result.';
|
||||
COMMENT ON TABLE proof_bundle IS 'Cryptographically-signed evidence bundles for audit trails. Contains DSSE-wrapped proof chains.';
|
||||
COMMENT ON TABLE score_replay_history IS 'History of score replays triggered by feed updates, policy changes, or manual requests.';
|
||||
@@ -0,0 +1,64 @@
|
||||
-- Migration: 007_unknowns_ranking_containment.sql
|
||||
-- Sprint: SPRINT_3600_0002_0001
|
||||
-- Task: UNK-RANK-005 - Add blast_radius, containment columns to unknowns table
|
||||
-- Description: Extend unknowns table with ranking signals for containment-aware scoring
|
||||
|
||||
-- Add blast radius columns
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS blast_dependents INT DEFAULT 0;
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS blast_net_facing BOOLEAN DEFAULT false;
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS blast_privilege TEXT DEFAULT 'user';
|
||||
|
||||
-- Add exploit pressure columns
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS epss DOUBLE PRECISION;
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS kev BOOLEAN DEFAULT false;
|
||||
|
||||
-- Add containment signal columns
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS containment_seccomp TEXT DEFAULT 'unknown';
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS containment_fs TEXT DEFAULT 'unknown';
|
||||
|
||||
-- Add proof reference for ranking explanation
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS proof_ref TEXT;
|
||||
|
||||
-- Add evidence scarcity column (0-1 range)
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS evidence_scarcity DOUBLE PRECISION DEFAULT 0.5;
|
||||
|
||||
-- Update score index for efficient sorting
|
||||
DROP INDEX IF EXISTS ix_unknowns_score_desc;
|
||||
CREATE INDEX IF NOT EXISTS ix_unknowns_score_desc ON unknowns(score DESC);
|
||||
|
||||
-- Composite index for common query patterns
|
||||
DROP INDEX IF EXISTS ix_unknowns_artifact_score;
|
||||
CREATE INDEX IF NOT EXISTS ix_unknowns_artifact_score ON unknowns(artifact_digest, score DESC);
|
||||
|
||||
-- Index for filtering by containment state
|
||||
DROP INDEX IF EXISTS ix_unknowns_containment;
|
||||
CREATE INDEX IF NOT EXISTS ix_unknowns_containment ON unknowns(containment_seccomp, containment_fs);
|
||||
|
||||
-- Index for KEV filtering (high priority unknowns)
|
||||
DROP INDEX IF EXISTS ix_unknowns_kev;
|
||||
CREATE INDEX IF NOT EXISTS ix_unknowns_kev ON unknowns(kev) WHERE kev = true;
|
||||
|
||||
-- Comments for documentation
|
||||
COMMENT ON COLUMN unknowns.blast_dependents IS 'Number of dependent packages affected by this unknown';
|
||||
COMMENT ON COLUMN unknowns.blast_net_facing IS 'Whether the affected code is network-facing';
|
||||
COMMENT ON COLUMN unknowns.blast_privilege IS 'Privilege level: root, user, unprivileged';
|
||||
COMMENT ON COLUMN unknowns.epss IS 'EPSS score if available (0.0-1.0)';
|
||||
COMMENT ON COLUMN unknowns.kev IS 'True if vulnerability is in CISA KEV catalog';
|
||||
COMMENT ON COLUMN unknowns.containment_seccomp IS 'Seccomp state: enforced, permissive, unknown';
|
||||
COMMENT ON COLUMN unknowns.containment_fs IS 'Filesystem state: ro (read-only), rw, unknown';
|
||||
COMMENT ON COLUMN unknowns.proof_ref IS 'Path to proof bundle explaining ranking factors';
|
||||
COMMENT ON COLUMN unknowns.evidence_scarcity IS 'Evidence scarcity factor (0=full evidence, 1=no evidence)';
|
||||
|
||||
-- Check constraint for valid privilege values
|
||||
ALTER TABLE unknowns DROP CONSTRAINT IF EXISTS chk_unknowns_privilege;
|
||||
ALTER TABLE unknowns ADD CONSTRAINT chk_unknowns_privilege
|
||||
CHECK (blast_privilege IN ('root', 'user', 'unprivileged'));
|
||||
|
||||
-- Check constraint for valid containment values
|
||||
ALTER TABLE unknowns DROP CONSTRAINT IF EXISTS chk_unknowns_seccomp;
|
||||
ALTER TABLE unknowns ADD CONSTRAINT chk_unknowns_seccomp
|
||||
CHECK (containment_seccomp IN ('enforced', 'permissive', 'unknown'));
|
||||
|
||||
ALTER TABLE unknowns DROP CONSTRAINT IF EXISTS chk_unknowns_fs;
|
||||
ALTER TABLE unknowns ADD CONSTRAINT chk_unknowns_fs
|
||||
CHECK (containment_fs IN ('ro', 'rw', 'unknown'));
|
||||
@@ -0,0 +1,292 @@
|
||||
-- SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
-- Sprint: Advisory-derived
|
||||
-- Task: EPSS Integration - Database Schema
|
||||
-- Description: Creates tables for EPSS (Exploit Prediction Scoring System) integration
|
||||
-- with time-series storage and change detection
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Import Provenance
|
||||
-- ============================================================================
|
||||
-- Tracks all EPSS import runs with full provenance for audit and replay
|
||||
CREATE TABLE IF NOT EXISTS epss_import_runs (
|
||||
import_run_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
model_date DATE NOT NULL,
|
||||
source_uri TEXT NOT NULL,
|
||||
retrieved_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
file_sha256 TEXT NOT NULL,
|
||||
decompressed_sha256 TEXT,
|
||||
row_count INT NOT NULL,
|
||||
model_version_tag TEXT, -- e.g., v2025.03.14 from leading # comment
|
||||
published_date DATE, -- from leading # comment if present
|
||||
status TEXT NOT NULL CHECK (status IN ('PENDING', 'SUCCEEDED', 'FAILED')),
|
||||
error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT epss_import_runs_model_date_unique UNIQUE (model_date)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_import_runs_model_date
|
||||
ON epss_import_runs (model_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_import_runs_status
|
||||
ON epss_import_runs (status);
|
||||
|
||||
COMMENT ON TABLE epss_import_runs IS 'Provenance tracking for all EPSS import operations';
|
||||
COMMENT ON COLUMN epss_import_runs.model_date IS 'The date of the EPSS model snapshot';
|
||||
COMMENT ON COLUMN epss_import_runs.source_uri IS 'Source URL or bundle:// URI for the import';
|
||||
COMMENT ON COLUMN epss_import_runs.file_sha256 IS 'SHA256 hash of the compressed file';
|
||||
COMMENT ON COLUMN epss_import_runs.decompressed_sha256 IS 'SHA256 hash of the decompressed CSV';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Time-Series Scores (Partitioned)
|
||||
-- ============================================================================
|
||||
-- Immutable append-only storage for all EPSS scores by date
|
||||
-- Partitioned by month for efficient querying and maintenance
|
||||
CREATE TABLE IF NOT EXISTS epss_scores (
|
||||
model_date DATE NOT NULL,
|
||||
cve_id TEXT NOT NULL,
|
||||
epss_score DOUBLE PRECISION NOT NULL CHECK (epss_score >= 0 AND epss_score <= 1),
|
||||
percentile DOUBLE PRECISION NOT NULL CHECK (percentile >= 0 AND percentile <= 1),
|
||||
import_run_id UUID NOT NULL REFERENCES epss_import_runs(import_run_id),
|
||||
PRIMARY KEY (model_date, cve_id)
|
||||
) PARTITION BY RANGE (model_date);
|
||||
|
||||
-- Create partitions for current and next 6 months
|
||||
-- Additional partitions should be created via scheduled maintenance
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2025_12 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2025-12-01') TO ('2026-01-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_01 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-01-01') TO ('2026-02-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_02 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-02-01') TO ('2026-03-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_03 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-03-01') TO ('2026-04-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_04 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-04-01') TO ('2026-05-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_05 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-05-01') TO ('2026-06-01');
|
||||
|
||||
-- Default partition for dates outside defined ranges
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_default PARTITION OF epss_scores DEFAULT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_scores_cve_id
|
||||
ON epss_scores (cve_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_scores_score_desc
|
||||
ON epss_scores (epss_score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_scores_cve_date
|
||||
ON epss_scores (cve_id, model_date DESC);
|
||||
|
||||
COMMENT ON TABLE epss_scores IS 'Immutable time-series storage for all EPSS scores';
|
||||
COMMENT ON COLUMN epss_scores.epss_score IS 'EPSS probability score (0.0 to 1.0)';
|
||||
COMMENT ON COLUMN epss_scores.percentile IS 'Percentile rank vs all CVEs (0.0 to 1.0)';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Current Projection (Fast Lookup)
|
||||
-- ============================================================================
|
||||
-- Materialized current EPSS for fast O(1) lookup
|
||||
-- Updated during each import after delta computation
|
||||
CREATE TABLE IF NOT EXISTS epss_current (
|
||||
cve_id TEXT PRIMARY KEY,
|
||||
epss_score DOUBLE PRECISION NOT NULL CHECK (epss_score >= 0 AND epss_score <= 1),
|
||||
percentile DOUBLE PRECISION NOT NULL CHECK (percentile >= 0 AND percentile <= 1),
|
||||
model_date DATE NOT NULL,
|
||||
import_run_id UUID NOT NULL REFERENCES epss_import_runs(import_run_id),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_current_score_desc
|
||||
ON epss_current (epss_score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_current_percentile_desc
|
||||
ON epss_current (percentile DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_current_model_date
|
||||
ON epss_current (model_date);
|
||||
|
||||
COMMENT ON TABLE epss_current IS 'Fast lookup projection of latest EPSS scores';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Change Detection (Partitioned)
|
||||
-- ============================================================================
|
||||
-- Tracks daily changes to enable efficient targeted enrichment
|
||||
CREATE TABLE IF NOT EXISTS epss_changes (
|
||||
model_date DATE NOT NULL,
|
||||
cve_id TEXT NOT NULL,
|
||||
old_score DOUBLE PRECISION,
|
||||
new_score DOUBLE PRECISION NOT NULL,
|
||||
delta_score DOUBLE PRECISION,
|
||||
old_percentile DOUBLE PRECISION,
|
||||
new_percentile DOUBLE PRECISION NOT NULL,
|
||||
delta_percentile DOUBLE PRECISION,
|
||||
flags INT NOT NULL DEFAULT 0,
|
||||
import_run_id UUID NOT NULL REFERENCES epss_import_runs(import_run_id),
|
||||
PRIMARY KEY (model_date, cve_id)
|
||||
) PARTITION BY RANGE (model_date);
|
||||
|
||||
-- Create partitions matching epss_scores
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2025_12 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2025-12-01') TO ('2026-01-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_01 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-01-01') TO ('2026-02-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_02 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-02-01') TO ('2026-03-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_03 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-03-01') TO ('2026-04-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_04 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-04-01') TO ('2026-05-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_05 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-05-01') TO ('2026-06-01');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_default PARTITION OF epss_changes DEFAULT;
|
||||
|
||||
-- Flags bitmask values:
|
||||
-- 0x01 = NEW_SCORED (CVE newly scored)
|
||||
-- 0x02 = CROSSED_HIGH (crossed above high score threshold)
|
||||
-- 0x04 = CROSSED_LOW (crossed below high score threshold)
|
||||
-- 0x08 = BIG_JUMP_UP (delta > 0.10 upward)
|
||||
-- 0x10 = BIG_JUMP_DOWN (delta > 0.10 downward)
|
||||
-- 0x20 = TOP_PERCENTILE (entered top 5%)
|
||||
-- 0x40 = LEFT_TOP_PERCENTILE (left top 5%)
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_changes_flags
|
||||
ON epss_changes (flags) WHERE flags > 0;
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_changes_delta
|
||||
ON epss_changes (ABS(delta_score) DESC) WHERE delta_score IS NOT NULL;
|
||||
|
||||
COMMENT ON TABLE epss_changes IS 'Daily change detection for targeted enrichment';
|
||||
COMMENT ON COLUMN epss_changes.flags IS 'Bitmask: 0x01=NEW, 0x02=CROSSED_HIGH, 0x04=CROSSED_LOW, 0x08=BIG_UP, 0x10=BIG_DOWN, 0x20=TOP_PCT';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Configuration
|
||||
-- ============================================================================
|
||||
-- Per-org or global thresholds for notification and scoring
|
||||
CREATE TABLE IF NOT EXISTS epss_config (
|
||||
config_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
org_id UUID, -- NULL for global defaults
|
||||
high_percentile DOUBLE PRECISION NOT NULL DEFAULT 0.95,
|
||||
high_score DOUBLE PRECISION NOT NULL DEFAULT 0.50,
|
||||
big_jump_delta DOUBLE PRECISION NOT NULL DEFAULT 0.10,
|
||||
score_weight DOUBLE PRECISION NOT NULL DEFAULT 0.25,
|
||||
notify_on_new_high BOOLEAN NOT NULL DEFAULT true,
|
||||
notify_on_crossing BOOLEAN NOT NULL DEFAULT true,
|
||||
notify_on_big_jump BOOLEAN NOT NULL DEFAULT true,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT epss_config_org_unique UNIQUE (org_id)
|
||||
);
|
||||
|
||||
-- Insert global defaults
|
||||
INSERT INTO epss_config (org_id, high_percentile, high_score, big_jump_delta, score_weight)
|
||||
VALUES (NULL, 0.95, 0.50, 0.10, 0.25)
|
||||
ON CONFLICT (org_id) DO NOTHING;
|
||||
|
||||
COMMENT ON TABLE epss_config IS 'EPSS notification and scoring thresholds';
|
||||
COMMENT ON COLUMN epss_config.high_percentile IS 'Threshold for top percentile alerts (default: 0.95 = top 5%)';
|
||||
COMMENT ON COLUMN epss_config.high_score IS 'Threshold for high score alerts (default: 0.50)';
|
||||
COMMENT ON COLUMN epss_config.big_jump_delta IS 'Threshold for significant daily change (default: 0.10)';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Evidence on Scan Findings
|
||||
-- ============================================================================
|
||||
-- Add EPSS-at-scan columns to existing scan_findings if not exists
|
||||
-- This preserves immutable evidence for replay
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'scan_findings' AND column_name = 'epss_score_at_scan'
|
||||
) THEN
|
||||
ALTER TABLE scan_findings
|
||||
ADD COLUMN epss_score_at_scan DOUBLE PRECISION,
|
||||
ADD COLUMN epss_percentile_at_scan DOUBLE PRECISION,
|
||||
ADD COLUMN epss_model_date_at_scan DATE,
|
||||
ADD COLUMN epss_import_run_id_at_scan UUID;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- ============================================================================
|
||||
-- Helper Functions
|
||||
-- ============================================================================
|
||||
|
||||
-- Function to compute change flags
|
||||
CREATE OR REPLACE FUNCTION compute_epss_change_flags(
|
||||
p_old_score DOUBLE PRECISION,
|
||||
p_new_score DOUBLE PRECISION,
|
||||
p_old_percentile DOUBLE PRECISION,
|
||||
p_new_percentile DOUBLE PRECISION,
|
||||
p_high_score DOUBLE PRECISION DEFAULT 0.50,
|
||||
p_high_percentile DOUBLE PRECISION DEFAULT 0.95,
|
||||
p_big_jump DOUBLE PRECISION DEFAULT 0.10
|
||||
) RETURNS INT AS $$
|
||||
DECLARE
|
||||
v_flags INT := 0;
|
||||
v_delta DOUBLE PRECISION;
|
||||
BEGIN
|
||||
-- NEW_SCORED
|
||||
IF p_old_score IS NULL THEN
|
||||
v_flags := v_flags | 1; -- 0x01
|
||||
END IF;
|
||||
|
||||
-- CROSSED_HIGH (score)
|
||||
IF p_old_score IS NOT NULL AND p_old_score < p_high_score AND p_new_score >= p_high_score THEN
|
||||
v_flags := v_flags | 2; -- 0x02
|
||||
END IF;
|
||||
|
||||
-- CROSSED_LOW (score)
|
||||
IF p_old_score IS NOT NULL AND p_old_score >= p_high_score AND p_new_score < p_high_score THEN
|
||||
v_flags := v_flags | 4; -- 0x04
|
||||
END IF;
|
||||
|
||||
-- BIG_JUMP_UP
|
||||
IF p_old_score IS NOT NULL THEN
|
||||
v_delta := p_new_score - p_old_score;
|
||||
IF v_delta > p_big_jump THEN
|
||||
v_flags := v_flags | 8; -- 0x08
|
||||
END IF;
|
||||
|
||||
-- BIG_JUMP_DOWN
|
||||
IF v_delta < -p_big_jump THEN
|
||||
v_flags := v_flags | 16; -- 0x10
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- TOP_PERCENTILE (entered)
|
||||
IF (p_old_percentile IS NULL OR p_old_percentile < p_high_percentile)
|
||||
AND p_new_percentile >= p_high_percentile THEN
|
||||
v_flags := v_flags | 32; -- 0x20
|
||||
END IF;
|
||||
|
||||
-- LEFT_TOP_PERCENTILE
|
||||
IF p_old_percentile IS NOT NULL AND p_old_percentile >= p_high_percentile
|
||||
AND p_new_percentile < p_high_percentile THEN
|
||||
v_flags := v_flags | 64; -- 0x40
|
||||
END IF;
|
||||
|
||||
RETURN v_flags;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
||||
|
||||
COMMENT ON FUNCTION compute_epss_change_flags IS 'Computes bitmask flags for EPSS change detection';
|
||||
|
||||
-- Function to create monthly partition
|
||||
CREATE OR REPLACE FUNCTION create_epss_partition(p_year INT, p_month INT)
|
||||
RETURNS VOID AS $$
|
||||
DECLARE
|
||||
v_start DATE;
|
||||
v_end DATE;
|
||||
v_partition_name TEXT;
|
||||
BEGIN
|
||||
v_start := make_date(p_year, p_month, 1);
|
||||
v_end := v_start + INTERVAL '1 month';
|
||||
v_partition_name := format('epss_scores_%s_%s', p_year, LPAD(p_month::TEXT, 2, '0'));
|
||||
|
||||
EXECUTE format(
|
||||
'CREATE TABLE IF NOT EXISTS %I PARTITION OF epss_scores FOR VALUES FROM (%L) TO (%L)',
|
||||
v_partition_name, v_start, v_end
|
||||
);
|
||||
|
||||
v_partition_name := format('epss_changes_%s_%s', p_year, LPAD(p_month::TEXT, 2, '0'));
|
||||
EXECUTE format(
|
||||
'CREATE TABLE IF NOT EXISTS %I PARTITION OF epss_changes FOR VALUES FROM (%L) TO (%L)',
|
||||
v_partition_name, v_start, v_end
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
COMMENT ON FUNCTION create_epss_partition IS 'Creates monthly partitions for EPSS tables';
|
||||
@@ -6,4 +6,8 @@ internal static class MigrationIds
|
||||
public const string ProofSpineTables = "002_proof_spine_tables.sql";
|
||||
public const string ClassificationHistory = "003_classification_history.sql";
|
||||
public const string ScanMetrics = "004_scan_metrics.sql";
|
||||
public const string SmartDiffTables = "005_smart_diff_tables.sql";
|
||||
public const string ScoreReplayTables = "006_score_replay_tables.sql";
|
||||
public const string UnknownsRankingContainment = "007_unknowns_ranking_containment.sql";
|
||||
public const string EpssIntegration = "008_epss_integration.sql";
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user