stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

feat(metrics): Implement scan metrics repository and PostgreSQL integration
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details

Browse Source 
- Added IScanMetricsRepository interface for scan metrics persistence and retrieval.
- Implemented PostgresScanMetricsRepository for PostgreSQL database interactions, including methods for saving and retrieving scan metrics and execution phases.
- Introduced methods for obtaining TTE statistics and recent scans for tenants.
- Implemented deletion of old metrics for retention purposes.

test(tests): Add SCA Failure Catalogue tests for FC6-FC10

- Created ScaCatalogueDeterminismTests to validate determinism properties of SCA Failure Catalogue fixtures.
- Developed ScaFailureCatalogueTests to ensure correct handling of specific failure modes in the scanner.
- Included tests for manifest validation, file existence, and expected findings across multiple failure cases.

feat(telemetry): Integrate scan completion metrics into the pipeline

- Introduced IScanCompletionMetricsIntegration interface and ScanCompletionMetricsIntegration class to record metrics upon scan completion.
- Implemented proof coverage and TTE metrics recording with logging for scan completion summaries.
This commit is contained in:
master
2025-12-16 14:00:35 +02:00
parent b55d9fa68d
commit 415eff1207
27 changed files with 3620 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

321
src/Scanner/StellaOps.Scanner.Worker/Metrics/ScanMetricsCollector.cs Normal file
View File

@@ -0,0 +1,321 @@
// -----------------------------------------------------------------------------
// ScanMetricsCollector.cs
// Sprint: SPRINT_3406_0001_0001_metrics_tables
// Task: METRICS-3406-008
// Description: Service for collecting and persisting scan metrics during execution
// -----------------------------------------------------------------------------
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Storage.Models;
using StellaOps.Scanner.Storage.Repositories;
namespace StellaOps.Scanner.Worker.Metrics;
/// <summary>
/// Collects and persists scan metrics during execution.
/// Thread-safe for concurrent phase tracking.
/// </summary>
public sealed class ScanMetricsCollector : IDisposable
{
private readonly IScanMetricsRepository _repository;
private readonly ILogger<ScanMetricsCollector> _logger;
private readonly Guid _scanId;
private readonly Guid _tenantId;
private readonly string _artifactDigest;
private readonly string _artifactType;
private readonly string _scannerVersion;
private readonly Stopwatch _totalStopwatch = new();
private readonly object _lock = new();
private readonly Dictionary<string, PhaseTracker> _phases = new();
private readonly List<ExecutionPhase> _completedPhases = [];
private DateTimeOffset _startedAt;
private Guid _metricsId;
private bool _disposed;
// Result tracking
private string? _findingsSha256;
private string? _vexBundleSha256;
private string? _proofBundleSha256;
private string? _sbomSha256;
private string? _policyDigest;
private string? _feedSnapshotId;
private int? _packageCount;
private int? _findingCount;
private int? _vexDecisionCount;
private Guid? _surfaceId;
private string? _replayManifestHash;
private string? _scannerImageDigest;
private bool _isReplay;
public ScanMetricsCollector(
IScanMetricsRepository repository,
ILogger<ScanMetricsCollector> logger,
Guid scanId,
Guid tenantId,
string artifactDigest,
string artifactType,
string scannerVersion)
{
_repository = repository ?? throw new ArgumentNullException(nameof(repository));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_scanId = scanId;
_tenantId = tenantId;
_artifactDigest = artifactDigest ?? throw new ArgumentNullException(nameof(artifactDigest));
_artifactType = artifactType ?? throw new ArgumentNullException(nameof(artifactType));
_scannerVersion = scannerVersion ?? throw new ArgumentNullException(nameof(scannerVersion));
_metricsId = Guid.NewGuid();
}
/// <summary>
/// Gets the metrics ID for this scan.
/// </summary>
public Guid MetricsId => _metricsId;
/// <summary>
/// Start collecting metrics.
/// </summary>
public void Start()
{
_startedAt = DateTimeOffset.UtcNow;
_totalStopwatch.Start();
_logger.LogDebug("Started metrics collection for scan {ScanId}", _scanId);
}
/// <summary>
/// Start tracking a phase.
/// </summary>
public IDisposable StartPhase(string phaseName)
{
lock (_lock)
{
if (_phases.ContainsKey(phaseName))
{
_logger.LogWarning("Phase {PhaseName} already started for scan {ScanId}", phaseName, _scanId);
return NoOpDisposable.Instance;
}
var tracker = new PhaseTracker(this, phaseName, DateTimeOffset.UtcNow);
_phases[phaseName] = tracker;
_logger.LogDebug("Started phase {PhaseName} for scan {ScanId}", phaseName, _scanId);
return tracker;
}
}
/// <summary>
/// Complete a phase with success.
/// </summary>
public void CompletePhase(string phaseName, Dictionary<string, object>? metrics = null)
{
CompletePhaseInternal(phaseName, success: true, errorCode: null, errorMessage: null, metrics);
}
/// <summary>
/// Complete a phase with failure.
/// </summary>
public void FailPhase(string phaseName, string errorCode, string? errorMessage = null)
{
CompletePhaseInternal(phaseName, success: false, errorCode, errorMessage, metrics: null);
}
private void CompletePhaseInternal(
string phaseName,
bool success,
string? errorCode,
string? errorMessage,
IReadOnlyDictionary<string, object>? metrics)
{
lock (_lock)
{
if (!_phases.TryGetValue(phaseName, out var tracker))
{
_logger.LogWarning("Phase {PhaseName} not started for scan {ScanId}", phaseName, _scanId);
return;
}
_phases.Remove(phaseName);
var finishedAt = DateTimeOffset.UtcNow;
var phase = new ExecutionPhase
{
MetricsId = _metricsId,
PhaseName = phaseName,
PhaseOrder = ScanPhaseNames.GetPhaseOrder(phaseName),
StartedAt = tracker.StartedAt,
FinishedAt = finishedAt,
Success = success,
ErrorCode = errorCode,
ErrorMessage = errorMessage,
PhaseMetrics = metrics
};
_completedPhases.Add(phase);
_logger.LogDebug(
"Completed phase {PhaseName} for scan {ScanId} in {DurationMs}ms (success={Success})",
phaseName, _scanId, phase.DurationMs, success);
}
}
/// <summary>
/// Set result digests.
/// </summary>
public void SetDigests(
string findingsSha256,
string? vexBundleSha256 = null,
string? proofBundleSha256 = null,
string? sbomSha256 = null)
{
_findingsSha256 = findingsSha256;
_vexBundleSha256 = vexBundleSha256;
_proofBundleSha256 = proofBundleSha256;
_sbomSha256 = sbomSha256;
}
/// <summary>
/// Set policy reference.
/// </summary>
public void SetPolicy(string? policyDigest, string? feedSnapshotId = null)
{
_policyDigest = policyDigest;
_feedSnapshotId = feedSnapshotId;
}
/// <summary>
/// Set artifact counts.
/// </summary>
public void SetCounts(int? packageCount = null, int? findingCount = null, int? vexDecisionCount = null)
{
_packageCount = packageCount;
_findingCount = findingCount;
_vexDecisionCount = vexDecisionCount;
}
/// <summary>
/// Set additional metadata.
/// </summary>
public void SetMetadata(
Guid? surfaceId = null,
string? replayManifestHash = null,
string? scannerImageDigest = null,
bool isReplay = false)
{
_surfaceId = surfaceId;
_replayManifestHash = replayManifestHash;
_scannerImageDigest = scannerImageDigest;
_isReplay = isReplay;
}
/// <summary>
/// Complete metrics collection and persist.
/// </summary>
public async Task CompleteAsync(CancellationToken cancellationToken = default)
{
_totalStopwatch.Stop();
var finishedAt = DateTimeOffset.UtcNow;
// Calculate phase timings
var phases = BuildPhaseTimings();
var metrics = new ScanMetrics
{
MetricsId = _metricsId,
ScanId = _scanId,
TenantId = _tenantId,
SurfaceId = _surfaceId,
ArtifactDigest = _artifactDigest,
ArtifactType = _artifactType,
ReplayManifestHash = _replayManifestHash,
FindingsSha256 = _findingsSha256 ?? string.Empty,
VexBundleSha256 = _vexBundleSha256,
ProofBundleSha256 = _proofBundleSha256,
SbomSha256 = _sbomSha256,
PolicyDigest = _policyDigest,
FeedSnapshotId = _feedSnapshotId,
StartedAt = _startedAt,
FinishedAt = finishedAt,
Phases = phases,
PackageCount = _packageCount,
FindingCount = _findingCount,
VexDecisionCount = _vexDecisionCount,
ScannerVersion = _scannerVersion,
ScannerImageDigest = _scannerImageDigest,
IsReplay = _isReplay
};
try
{
await _repository.SaveAsync(metrics, cancellationToken);
await _repository.SavePhasesAsync(_completedPhases, cancellationToken);
_logger.LogInformation(
"Saved metrics for scan {ScanId}: TTE={TteMms}ms, Packages={Packages}, Findings={Findings}",
_scanId, metrics.TotalDurationMs, _packageCount, _findingCount);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to save metrics for scan {ScanId}", _scanId);
throw;
}
}
private ScanPhaseTimings BuildPhaseTimings()
{
lock (_lock)
{
int GetPhaseDuration(string name) =>
_completedPhases
.Where(p => p.PhaseName == name)
.Sum(p => p.DurationMs);
return new ScanPhaseTimings
{
IngestMs = GetPhaseDuration(ScanPhaseNames.Ingest),
AnalyzeMs = GetPhaseDuration(ScanPhaseNames.Analyze),
ReachabilityMs = GetPhaseDuration(ScanPhaseNames.Reachability),
VexMs = GetPhaseDuration(ScanPhaseNames.Vex),
SignMs = GetPhaseDuration(ScanPhaseNames.Sign),
PublishMs = GetPhaseDuration(ScanPhaseNames.Publish)
};
}
}
public void Dispose()
{
if (_disposed) return;
_disposed = true;
_totalStopwatch.Stop();
}
private sealed class PhaseTracker : IDisposable
{
private readonly ScanMetricsCollector _collector;
private readonly string _phaseName;
private bool _disposed;
public DateTimeOffset StartedAt { get; }
public PhaseTracker(ScanMetricsCollector collector, string phaseName, DateTimeOffset startedAt)
{
_collector = collector;
_phaseName = phaseName;
StartedAt = startedAt;
}
public void Dispose()
{
if (_disposed) return;
_disposed = true;
_collector.CompletePhase(_phaseName);
}
}
private sealed class NoOpDisposable : IDisposable
{
public static readonly NoOpDisposable Instance = new();
private NoOpDisposable() { }
public void Dispose() { }
}
}

173
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Models/ScanMetricsModels.cs Normal file
View File

@@ -0,0 +1,173 @@
// -----------------------------------------------------------------------------
// ScanMetricsModels.cs
// Sprint: SPRINT_3406_0001_0001_metrics_tables
// Task: METRICS-3406-005
// Description: Entity definitions for scan metrics and TTE tracking
// -----------------------------------------------------------------------------
namespace StellaOps.Scanner.Storage.Models;
/// <summary>
/// Per-scan metrics for TTE tracking.
/// </summary>
public sealed record ScanMetrics
{
public Guid MetricsId { get; init; }
public required Guid ScanId { get; init; }
public required Guid TenantId { get; init; }
public Guid? SurfaceId { get; init; }
// Artifact identification
public required string ArtifactDigest { get; init; }
public required string ArtifactType { get; init; }
// Reference to replay manifest
public string? ReplayManifestHash { get; init; }
// Digest tracking
public required string FindingsSha256 { get; init; }
public string? VexBundleSha256 { get; init; }
public string? ProofBundleSha256 { get; init; }
public string? SbomSha256 { get; init; }
// Policy reference
public string? PolicyDigest { get; init; }
public string? FeedSnapshotId { get; init; }
// Timing
public required DateTimeOffset StartedAt { get; init; }
public required DateTimeOffset FinishedAt { get; init; }
/// <summary>
/// Time-to-Evidence in milliseconds.
/// </summary>
public int TotalDurationMs => (int)(FinishedAt - StartedAt).TotalMilliseconds;
// Phase timings
public required ScanPhaseTimings Phases { get; init; }
// Artifact counts
public int? PackageCount { get; init; }
public int? FindingCount { get; init; }
public int? VexDecisionCount { get; init; }
// Scanner metadata
public required string ScannerVersion { get; init; }
public string? ScannerImageDigest { get; init; }
// Replay mode
public bool IsReplay { get; init; }
public DateTimeOffset CreatedAt { get; init; } = DateTimeOffset.UtcNow;
}
/// <summary>
/// Phase timing breakdown (milliseconds).
/// </summary>
public sealed record ScanPhaseTimings
{
public required int IngestMs { get; init; }
public required int AnalyzeMs { get; init; }
public required int ReachabilityMs { get; init; }
public required int VexMs { get; init; }
public required int SignMs { get; init; }
public required int PublishMs { get; init; }
/// <summary>
/// Sum of all phases.
/// </summary>
public int TotalMs => IngestMs + AnalyzeMs + ReachabilityMs + VexMs + SignMs + PublishMs;
/// <summary>
/// Create empty timing record.
/// </summary>
public static ScanPhaseTimings Empty => new()
{
IngestMs = 0,
AnalyzeMs = 0,
ReachabilityMs = 0,
VexMs = 0,
SignMs = 0,
PublishMs = 0
};
}
/// <summary>
/// Detailed phase execution record.
/// </summary>
public sealed record ExecutionPhase
{
public long Id { get; init; }
public required Guid MetricsId { get; init; }
public required string PhaseName { get; init; }
public required int PhaseOrder { get; init; }
public required DateTimeOffset StartedAt { get; init; }
public required DateTimeOffset FinishedAt { get; init; }
public int DurationMs => (int)(FinishedAt - StartedAt).TotalMilliseconds;
public required bool Success { get; init; }
public string? ErrorCode { get; init; }
public string? ErrorMessage { get; init; }
public IReadOnlyDictionary<string, object>? PhaseMetrics { get; init; }
}
/// <summary>
/// TTE statistics for a time period.
/// </summary>
public sealed record TteStats
{
public required Guid TenantId { get; init; }
public required DateTimeOffset HourBucket { get; init; }
public required int ScanCount { get; init; }
public required int TteAvgMs { get; init; }
public required int TteP50Ms { get; init; }
public required int TteP95Ms { get; init; }
public required int TteMaxMs { get; init; }
public required decimal SloP50CompliancePercent { get; init; }
public required decimal SloP95CompliancePercent { get; init; }
}
/// <summary>
/// Standard scan phase names.
/// </summary>
public static class ScanPhaseNames
{
public const string Ingest = "ingest";
public const string Analyze = "analyze";
public const string Reachability = "reachability";
public const string Vex = "vex";
public const string Sign = "sign";
public const string Publish = "publish";
public const string Other = "other";
public static readonly IReadOnlyList<string> All =
[
Ingest,
Analyze,
Reachability,
Vex,
Sign,
Publish
];
public static int GetPhaseOrder(string phaseName) => phaseName switch
{
Ingest => 1,
Analyze => 2,
Reachability => 3,
Vex => 4,
Sign => 5,
Publish => 6,
_ => 99
};
}
/// <summary>
/// Artifact type constants.
/// </summary>
public static class ArtifactTypes
{
public const string OciImage = "oci_image";
public const string Tarball = "tarball";
public const string Directory = "directory";
public const string Other = "other";
}

208
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/Migrations/004_scan_metrics.sql Normal file
View File

@@ -0,0 +1,208 @@
-- Migration: 004_scan_metrics
-- Sprint: SPRINT_3406_0001_0001_metrics_tables
-- Task: METRICS-3406-001, METRICS-3406-002, METRICS-3406-003, METRICS-3406-004
-- Description: Scan metrics tables for TTE tracking and performance analysis
-- Create scanner schema if not exists
CREATE SCHEMA IF NOT EXISTS scanner;
-- =============================================================================
-- Task METRICS-3406-001: scan_metrics Table
-- =============================================================================
CREATE TABLE IF NOT EXISTS scanner.scan_metrics (
metrics_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-- Scan identification
scan_id UUID NOT NULL UNIQUE,
tenant_id UUID NOT NULL,
surface_id UUID,
-- Artifact identification
artifact_digest TEXT NOT NULL,
artifact_type TEXT NOT NULL, -- 'oci_image', 'tarball', 'directory'
-- Reference to replay manifest (in document store)
replay_manifest_hash TEXT,
-- Digest tracking for determinism
findings_sha256 TEXT NOT NULL,
vex_bundle_sha256 TEXT,
proof_bundle_sha256 TEXT,
sbom_sha256 TEXT,
-- Policy reference
policy_digest TEXT,
feed_snapshot_id TEXT,
-- Overall timing
started_at TIMESTAMPTZ NOT NULL,
finished_at TIMESTAMPTZ NOT NULL,
total_duration_ms INT NOT NULL GENERATED ALWAYS AS (
EXTRACT(EPOCH FROM (finished_at - started_at)) * 1000
) STORED,
-- Phase timings (milliseconds)
t_ingest_ms INT NOT NULL DEFAULT 0,
t_analyze_ms INT NOT NULL DEFAULT 0,
t_reachability_ms INT NOT NULL DEFAULT 0,
t_vex_ms INT NOT NULL DEFAULT 0,
t_sign_ms INT NOT NULL DEFAULT 0,
t_publish_ms INT NOT NULL DEFAULT 0,
-- Artifact counts
package_count INT,
finding_count INT,
vex_decision_count INT,
-- Scanner metadata
scanner_version TEXT NOT NULL,
scanner_image_digest TEXT,
-- Replay mode flag
is_replay BOOLEAN NOT NULL DEFAULT FALSE,
-- Timestamps
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-- Constraints
CONSTRAINT valid_timings CHECK (
t_ingest_ms >= 0 AND t_analyze_ms >= 0 AND t_reachability_ms >= 0 AND
t_vex_ms >= 0 AND t_sign_ms >= 0 AND t_publish_ms >= 0
),
CONSTRAINT valid_artifact_type CHECK (artifact_type IN ('oci_image', 'tarball', 'directory', 'other'))
);
COMMENT ON TABLE scanner.scan_metrics IS 'Per-scan metrics for TTE analysis and performance tracking';
COMMENT ON COLUMN scanner.scan_metrics.total_duration_ms IS 'Time-to-Evidence in milliseconds';
-- =============================================================================
-- Task METRICS-3406-002: execution_phases Table
-- =============================================================================
CREATE TABLE IF NOT EXISTS scanner.execution_phases (
id BIGSERIAL PRIMARY KEY,
metrics_id UUID NOT NULL REFERENCES scanner.scan_metrics(metrics_id) ON DELETE CASCADE,
-- Phase identification
phase_name TEXT NOT NULL,
phase_order INT NOT NULL,
-- Timing
started_at TIMESTAMPTZ NOT NULL,
finished_at TIMESTAMPTZ NOT NULL,
duration_ms INT NOT NULL GENERATED ALWAYS AS (
EXTRACT(EPOCH FROM (finished_at - started_at)) * 1000
) STORED,
-- Status
success BOOLEAN NOT NULL,
error_code TEXT,
error_message TEXT,
-- Phase-specific metrics (JSONB for flexibility)
phase_metrics JSONB,
-- Constraints
CONSTRAINT valid_phase_name CHECK (phase_name IN (
'ingest', 'analyze', 'reachability', 'vex', 'sign', 'publish', 'other'
))
);
COMMENT ON TABLE scanner.execution_phases IS 'Granular phase-level execution details';
-- =============================================================================
-- Task METRICS-3406-004: Indexes
-- =============================================================================
CREATE INDEX IF NOT EXISTS idx_scan_metrics_tenant ON scanner.scan_metrics(tenant_id);
CREATE INDEX IF NOT EXISTS idx_scan_metrics_artifact ON scanner.scan_metrics(artifact_digest);
CREATE INDEX IF NOT EXISTS idx_scan_metrics_started ON scanner.scan_metrics(started_at);
CREATE INDEX IF NOT EXISTS idx_scan_metrics_surface ON scanner.scan_metrics(surface_id);
CREATE INDEX IF NOT EXISTS idx_scan_metrics_replay ON scanner.scan_metrics(is_replay);
CREATE INDEX IF NOT EXISTS idx_scan_metrics_tenant_started ON scanner.scan_metrics(tenant_id, started_at);
CREATE INDEX IF NOT EXISTS idx_execution_phases_metrics ON scanner.execution_phases(metrics_id);
CREATE INDEX IF NOT EXISTS idx_execution_phases_name ON scanner.execution_phases(phase_name);
-- =============================================================================
-- Task METRICS-3406-003: scan_tte View
-- =============================================================================
CREATE OR REPLACE VIEW scanner.scan_tte AS
SELECT
metrics_id,
scan_id,
tenant_id,
surface_id,
artifact_digest,
-- TTE calculation
total_duration_ms AS tte_ms,
(total_duration_ms / 1000.0) AS tte_seconds,
(finished_at - started_at) AS tte_interval,
-- Phase breakdown
t_ingest_ms,
t_analyze_ms,
t_reachability_ms,
t_vex_ms,
t_sign_ms,
t_publish_ms,
-- Phase percentages
ROUND((t_ingest_ms::numeric / NULLIF(total_duration_ms, 0)) * 100, 2) AS ingest_percent,
ROUND((t_analyze_ms::numeric / NULLIF(total_duration_ms, 0)) * 100, 2) AS analyze_percent,
ROUND((t_reachability_ms::numeric / NULLIF(total_duration_ms, 0)) * 100, 2) AS reachability_percent,
ROUND((t_vex_ms::numeric / NULLIF(total_duration_ms, 0)) * 100, 2) AS vex_percent,
ROUND((t_sign_ms::numeric / NULLIF(total_duration_ms, 0)) * 100, 2) AS sign_percent,
ROUND((t_publish_ms::numeric / NULLIF(total_duration_ms, 0)) * 100, 2) AS publish_percent,
-- Metadata
package_count,
finding_count,
is_replay,
scanner_version,
started_at,
finished_at
FROM scanner.scan_metrics;
COMMENT ON VIEW scanner.scan_tte IS 'Time-to-Evidence metrics per scan';
-- TTE percentile calculation function
CREATE OR REPLACE FUNCTION scanner.tte_percentile(
p_tenant_id UUID,
p_percentile NUMERIC,
p_since TIMESTAMPTZ DEFAULT (NOW() - INTERVAL '7 days')
)
RETURNS NUMERIC AS $$
SELECT PERCENTILE_CONT(p_percentile) WITHIN GROUP (ORDER BY tte_ms)
FROM scanner.scan_tte
WHERE tenant_id = p_tenant_id
AND started_at >= p_since
AND NOT is_replay;
$$ LANGUAGE SQL STABLE;
-- TTE statistics aggregation view
CREATE OR REPLACE VIEW scanner.tte_stats AS
SELECT
tenant_id,
date_trunc('hour', started_at) AS hour_bucket,
COUNT(*) AS scan_count,
-- TTE statistics (ms)
AVG(tte_ms)::INT AS tte_avg_ms,
PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY tte_ms)::INT AS tte_p50_ms,
PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY tte_ms)::INT AS tte_p95_ms,
MAX(tte_ms) AS tte_max_ms,
-- SLO compliance (P50 < 120s = 120000ms, P95 < 300s = 300000ms)
ROUND(
(COUNT(*) FILTER (WHERE tte_ms < 120000)::numeric / COUNT(*)) * 100, 2
) AS slo_p50_compliance_percent,
ROUND(
(COUNT(*) FILTER (WHERE tte_ms < 300000)::numeric / COUNT(*)) * 100, 2
) AS slo_p95_compliance_percent
FROM scanner.scan_tte
WHERE NOT is_replay
GROUP BY tenant_id, date_trunc('hour', started_at);
COMMENT ON VIEW scanner.tte_stats IS 'Hourly TTE statistics with SLO compliance';

1
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/Migrations/MigrationIds.cs
View File

@@ -5,4 +5,5 @@ internal static class MigrationIds
public const string CreateTables = "001_create_tables.sql";
public const string ProofSpineTables = "002_proof_spine_tables.sql";
public const string ClassificationHistory = "003_classification_history.sql";
public const string ScanMetrics = "004_scan_metrics.sql";
}

85
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/IScanMetricsRepository.cs Normal file
View File

@@ -0,0 +1,85 @@
// -----------------------------------------------------------------------------
// IScanMetricsRepository.cs
// Sprint: SPRINT_3406_0001_0001_metrics_tables
// Task: METRICS-3406-006
// Description: Repository interface for scan metrics persistence
// -----------------------------------------------------------------------------
using StellaOps.Scanner.Storage.Models;
namespace StellaOps.Scanner.Storage.Repositories;
/// <summary>
/// Repository for scan metrics persistence and retrieval.
/// </summary>
public interface IScanMetricsRepository
{
/// <summary>
/// Save scan metrics after scan completion.
/// </summary>
Task SaveAsync(ScanMetrics metrics, CancellationToken cancellationToken = default);
/// <summary>
/// Save execution phase details.
/// </summary>
Task SavePhaseAsync(ExecutionPhase phase, CancellationToken cancellationToken = default);
/// <summary>
/// Save multiple execution phases.
/// </summary>
Task SavePhasesAsync(IReadOnlyList<ExecutionPhase> phases, CancellationToken cancellationToken = default);
/// <summary>
/// Get metrics by scan ID.
/// </summary>
Task<ScanMetrics?> GetByScanIdAsync(Guid scanId, CancellationToken cancellationToken = default);
/// <summary>
/// Get metrics by metrics ID.
/// </summary>
Task<ScanMetrics?> GetByIdAsync(Guid metricsId, CancellationToken cancellationToken = default);
/// <summary>
/// Get execution phases for a scan.
/// </summary>
Task<IReadOnlyList<ExecutionPhase>> GetPhasesAsync(Guid metricsId, CancellationToken cancellationToken = default);
/// <summary>
/// Get TTE statistics for a tenant within a time range.
/// </summary>
Task<IReadOnlyList<TteStats>> GetTteStatsAsync(
Guid tenantId,
DateTimeOffset since,
DateTimeOffset until,
CancellationToken cancellationToken = default);
/// <summary>
/// Get TTE percentile for a tenant.
/// </summary>
Task<int?> GetTtePercentileAsync(
Guid tenantId,
decimal percentile,
DateTimeOffset since,
CancellationToken cancellationToken = default);
/// <summary>
/// Get recent scans for a tenant.
/// </summary>
Task<IReadOnlyList<ScanMetrics>> GetRecentAsync(
Guid tenantId,
int limit = 100,
bool includeReplays = false,
CancellationToken cancellationToken = default);
/// <summary>
/// Get scans by artifact digest.
/// </summary>
Task<IReadOnlyList<ScanMetrics>> GetByArtifactAsync(
string artifactDigest,
CancellationToken cancellationToken = default);
/// <summary>
/// Delete old metrics (for retention).
/// </summary>
Task<int> DeleteOlderThanAsync(DateTimeOffset threshold, CancellationToken cancellationToken = default);
}

445
src/Scanner/__Libraries/StellaOps.Scanner.Storage/Repositories/PostgresScanMetricsRepository.cs Normal file
View File

@@ -0,0 +1,445 @@
// -----------------------------------------------------------------------------
// PostgresScanMetricsRepository.cs
// Sprint: SPRINT_3406_0001_0001_metrics_tables
// Task: METRICS-3406-007
// Description: PostgreSQL implementation of scan metrics repository
// -----------------------------------------------------------------------------
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Npgsql;
using StellaOps.Scanner.Storage.Models;
namespace StellaOps.Scanner.Storage.Repositories;
/// <summary>
/// PostgreSQL implementation of <see cref="IScanMetricsRepository"/>.
/// </summary>
public sealed class PostgresScanMetricsRepository : IScanMetricsRepository
{
private readonly NpgsqlDataSource _dataSource;
private readonly ILogger<PostgresScanMetricsRepository> _logger;
public PostgresScanMetricsRepository(
NpgsqlDataSource dataSource,
ILogger<PostgresScanMetricsRepository> logger)
{
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <inheritdoc/>
public async Task SaveAsync(ScanMetrics metrics, CancellationToken cancellationToken = default)
{
const string sql = """
INSERT INTO scanner.scan_metrics (
metrics_id, scan_id, tenant_id, surface_id,
artifact_digest, artifact_type, replay_manifest_hash,
findings_sha256, vex_bundle_sha256, proof_bundle_sha256, sbom_sha256,
policy_digest, feed_snapshot_id,
started_at, finished_at,
t_ingest_ms, t_analyze_ms, t_reachability_ms, t_vex_ms, t_sign_ms, t_publish_ms,
package_count, finding_count, vex_decision_count,
scanner_version, scanner_image_digest, is_replay, created_at
) VALUES (
@metricsId, @scanId, @tenantId, @surfaceId,
@artifactDigest, @artifactType, @replayManifestHash,
@findingsSha256, @vexBundleSha256, @proofBundleSha256, @sbomSha256,
@policyDigest, @feedSnapshotId,
@startedAt, @finishedAt,
@tIngestMs, @tAnalyzeMs, @tReachabilityMs, @tVexMs, @tSignMs, @tPublishMs,
@packageCount, @findingCount, @vexDecisionCount,
@scannerVersion, @scannerImageDigest, @isReplay, @createdAt
)
ON CONFLICT (scan_id) DO UPDATE SET
finished_at = EXCLUDED.finished_at,
t_ingest_ms = EXCLUDED.t_ingest_ms,
t_analyze_ms = EXCLUDED.t_analyze_ms,
t_reachability_ms = EXCLUDED.t_reachability_ms,
t_vex_ms = EXCLUDED.t_vex_ms,
t_sign_ms = EXCLUDED.t_sign_ms,
t_publish_ms = EXCLUDED.t_publish_ms,
findings_sha256 = EXCLUDED.findings_sha256,
package_count = EXCLUDED.package_count,
finding_count = EXCLUDED.finding_count,
vex_decision_count = EXCLUDED.vex_decision_count
""";
await using var cmd = _dataSource.CreateCommand(sql);
var metricsId = metrics.MetricsId == Guid.Empty ? Guid.NewGuid() : metrics.MetricsId;
cmd.Parameters.AddWithValue("metricsId", metricsId);
cmd.Parameters.AddWithValue("scanId", metrics.ScanId);
cmd.Parameters.AddWithValue("tenantId", metrics.TenantId);
cmd.Parameters.AddWithValue("surfaceId", (object?)metrics.SurfaceId ?? DBNull.Value);
cmd.Parameters.AddWithValue("artifactDigest", metrics.ArtifactDigest);
cmd.Parameters.AddWithValue("artifactType", metrics.ArtifactType);
cmd.Parameters.AddWithValue("replayManifestHash", (object?)metrics.ReplayManifestHash ?? DBNull.Value);
cmd.Parameters.AddWithValue("findingsSha256", metrics.FindingsSha256);
cmd.Parameters.AddWithValue("vexBundleSha256", (object?)metrics.VexBundleSha256 ?? DBNull.Value);
cmd.Parameters.AddWithValue("proofBundleSha256", (object?)metrics.ProofBundleSha256 ?? DBNull.Value);
cmd.Parameters.AddWithValue("sbomSha256", (object?)metrics.SbomSha256 ?? DBNull.Value);
cmd.Parameters.AddWithValue("policyDigest", (object?)metrics.PolicyDigest ?? DBNull.Value);
cmd.Parameters.AddWithValue("feedSnapshotId", (object?)metrics.FeedSnapshotId ?? DBNull.Value);
cmd.Parameters.AddWithValue("startedAt", metrics.StartedAt);
cmd.Parameters.AddWithValue("finishedAt", metrics.FinishedAt);
cmd.Parameters.AddWithValue("tIngestMs", metrics.Phases.IngestMs);
cmd.Parameters.AddWithValue("tAnalyzeMs", metrics.Phases.AnalyzeMs);
cmd.Parameters.AddWithValue("tReachabilityMs", metrics.Phases.ReachabilityMs);
cmd.Parameters.AddWithValue("tVexMs", metrics.Phases.VexMs);
cmd.Parameters.AddWithValue("tSignMs", metrics.Phases.SignMs);
cmd.Parameters.AddWithValue("tPublishMs", metrics.Phases.PublishMs);
cmd.Parameters.AddWithValue("packageCount", (object?)metrics.PackageCount ?? DBNull.Value);
cmd.Parameters.AddWithValue("findingCount", (object?)metrics.FindingCount ?? DBNull.Value);
cmd.Parameters.AddWithValue("vexDecisionCount", (object?)metrics.VexDecisionCount ?? DBNull.Value);
cmd.Parameters.AddWithValue("scannerVersion", metrics.ScannerVersion);
cmd.Parameters.AddWithValue("scannerImageDigest", (object?)metrics.ScannerImageDigest ?? DBNull.Value);
cmd.Parameters.AddWithValue("isReplay", metrics.IsReplay);
cmd.Parameters.AddWithValue("createdAt", metrics.CreatedAt);
await cmd.ExecuteNonQueryAsync(cancellationToken);
_logger.LogDebug("Saved scan metrics for scan {ScanId}", metrics.ScanId);
}
/// <inheritdoc/>
public async Task SavePhaseAsync(ExecutionPhase phase, CancellationToken cancellationToken = default)
{
await SavePhasesAsync([phase], cancellationToken);
}
/// <inheritdoc/>
public async Task SavePhasesAsync(IReadOnlyList<ExecutionPhase> phases, CancellationToken cancellationToken = default)
{
if (phases.Count == 0) return;
const string sql = """
INSERT INTO scanner.execution_phases (
metrics_id, phase_name, phase_order,
started_at, finished_at, success,
error_code, error_message, phase_metrics
) VALUES (
@metricsId, @phaseName, @phaseOrder,
@startedAt, @finishedAt, @success,
@errorCode, @errorMessage, @phaseMetrics::jsonb
)
""";
await using var connection = await _dataSource.OpenConnectionAsync(cancellationToken);
await using var transaction = await connection.BeginTransactionAsync(cancellationToken);
try
{
foreach (var phase in phases)
{
await using var cmd = new NpgsqlCommand(sql, connection, transaction);
cmd.Parameters.AddWithValue("metricsId", phase.MetricsId);
cmd.Parameters.AddWithValue("phaseName", phase.PhaseName);
cmd.Parameters.AddWithValue("phaseOrder", phase.PhaseOrder);
cmd.Parameters.AddWithValue("startedAt", phase.StartedAt);
cmd.Parameters.AddWithValue("finishedAt", phase.FinishedAt);
cmd.Parameters.AddWithValue("success", phase.Success);
cmd.Parameters.AddWithValue("errorCode", (object?)phase.ErrorCode ?? DBNull.Value);
cmd.Parameters.AddWithValue("errorMessage", (object?)phase.ErrorMessage ?? DBNull.Value);
cmd.Parameters.AddWithValue("phaseMetrics",
phase.PhaseMetrics is not null
? JsonSerializer.Serialize(phase.PhaseMetrics)
: DBNull.Value);
await cmd.ExecuteNonQueryAsync(cancellationToken);
}
await transaction.CommitAsync(cancellationToken);
}
catch
{
await transaction.RollbackAsync(cancellationToken);
throw;
}
}
/// <inheritdoc/>
public async Task<ScanMetrics?> GetByScanIdAsync(Guid scanId, CancellationToken cancellationToken = default)
{
const string sql = """
SELECT * FROM scanner.scan_metrics WHERE scan_id = @scanId
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("scanId", scanId);
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
if (await reader.ReadAsync(cancellationToken))
{
return MapToScanMetrics(reader);
}
return null;
}
/// <inheritdoc/>
public async Task<ScanMetrics?> GetByIdAsync(Guid metricsId, CancellationToken cancellationToken = default)
{
const string sql = """
SELECT * FROM scanner.scan_metrics WHERE metrics_id = @metricsId
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("metricsId", metricsId);
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
if (await reader.ReadAsync(cancellationToken))
{
return MapToScanMetrics(reader);
}
return null;
}
/// <inheritdoc/>
public async Task<IReadOnlyList<ExecutionPhase>> GetPhasesAsync(Guid metricsId, CancellationToken cancellationToken = default)
{
const string sql = """
SELECT * FROM scanner.execution_phases
WHERE metrics_id = @metricsId
ORDER BY phase_order
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("metricsId", metricsId);
var phases = new List<ExecutionPhase>();
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken))
{
phases.Add(MapToExecutionPhase(reader));
}
return phases;
}
/// <inheritdoc/>
public async Task<IReadOnlyList<TteStats>> GetTteStatsAsync(
Guid tenantId,
DateTimeOffset since,
DateTimeOffset until,
CancellationToken cancellationToken = default)
{
const string sql = """
SELECT * FROM scanner.tte_stats
WHERE tenant_id = @tenantId
AND hour_bucket >= @since
AND hour_bucket < @until
ORDER BY hour_bucket
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("tenantId", tenantId);
cmd.Parameters.AddWithValue("since", since);
cmd.Parameters.AddWithValue("until", until);
var stats = new List<TteStats>();
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken))
{
stats.Add(MapToTteStats(reader));
}
return stats;
}
/// <inheritdoc/>
public async Task<int?> GetTtePercentileAsync(
Guid tenantId,
decimal percentile,
DateTimeOffset since,
CancellationToken cancellationToken = default)
{
const string sql = """
SELECT scanner.tte_percentile(@tenantId, @percentile, @since)
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("tenantId", tenantId);
cmd.Parameters.AddWithValue("percentile", percentile);
cmd.Parameters.AddWithValue("since", since);
var result = await cmd.ExecuteScalarAsync(cancellationToken);
return result is DBNull or null ? null : Convert.ToInt32(result);
}
/// <inheritdoc/>
public async Task<IReadOnlyList<ScanMetrics>> GetRecentAsync(
Guid tenantId,
int limit = 100,
bool includeReplays = false,
CancellationToken cancellationToken = default)
{
var sql = $"""
SELECT * FROM scanner.scan_metrics
WHERE tenant_id = @tenantId
{(includeReplays ? "" : "AND NOT is_replay")}
ORDER BY started_at DESC
LIMIT @limit
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("tenantId", tenantId);
cmd.Parameters.AddWithValue("limit", limit);
var metrics = new List<ScanMetrics>();
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken))
{
metrics.Add(MapToScanMetrics(reader));
}
return metrics;
}
/// <inheritdoc/>
public async Task<IReadOnlyList<ScanMetrics>> GetByArtifactAsync(
string artifactDigest,
CancellationToken cancellationToken = default)
{
const string sql = """
SELECT * FROM scanner.scan_metrics
WHERE artifact_digest = @artifactDigest
ORDER BY started_at DESC
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("artifactDigest", artifactDigest);
var metrics = new List<ScanMetrics>();
await using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
while (await reader.ReadAsync(cancellationToken))
{
metrics.Add(MapToScanMetrics(reader));
}
return metrics;
}
/// <inheritdoc/>
public async Task<int> DeleteOlderThanAsync(DateTimeOffset threshold, CancellationToken cancellationToken = default)
{
const string sql = """
DELETE FROM scanner.scan_metrics WHERE started_at < @threshold
""";
await using var cmd = _dataSource.CreateCommand(sql);
cmd.Parameters.AddWithValue("threshold", threshold);
return await cmd.ExecuteNonQueryAsync(cancellationToken);
}
private static ScanMetrics MapToScanMetrics(NpgsqlDataReader reader)
{
return new ScanMetrics
{
MetricsId = reader.GetGuid(reader.GetOrdinal("metrics_id")),
ScanId = reader.GetGuid(reader.GetOrdinal("scan_id")),
TenantId = reader.GetGuid(reader.GetOrdinal("tenant_id")),
SurfaceId = reader.IsDBNull(reader.GetOrdinal("surface_id"))
? null
: reader.GetGuid(reader.GetOrdinal("surface_id")),
ArtifactDigest = reader.GetString(reader.GetOrdinal("artifact_digest")),
ArtifactType = reader.GetString(reader.GetOrdinal("artifact_type")),
ReplayManifestHash = reader.IsDBNull(reader.GetOrdinal("replay_manifest_hash"))
? null
: reader.GetString(reader.GetOrdinal("replay_manifest_hash")),
FindingsSha256 = reader.GetString(reader.GetOrdinal("findings_sha256")),
VexBundleSha256 = reader.IsDBNull(reader.GetOrdinal("vex_bundle_sha256"))
? null
: reader.GetString(reader.GetOrdinal("vex_bundle_sha256")),
ProofBundleSha256 = reader.IsDBNull(reader.GetOrdinal("proof_bundle_sha256"))
? null
: reader.GetString(reader.GetOrdinal("proof_bundle_sha256")),
SbomSha256 = reader.IsDBNull(reader.GetOrdinal("sbom_sha256"))
? null
: reader.GetString(reader.GetOrdinal("sbom_sha256")),
PolicyDigest = reader.IsDBNull(reader.GetOrdinal("policy_digest"))
? null
: reader.GetString(reader.GetOrdinal("policy_digest")),
FeedSnapshotId = reader.IsDBNull(reader.GetOrdinal("feed_snapshot_id"))
? null
: reader.GetString(reader.GetOrdinal("feed_snapshot_id")),
StartedAt = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("started_at")),
FinishedAt = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("finished_at")),
Phases = new ScanPhaseTimings
{
IngestMs = reader.GetInt32(reader.GetOrdinal("t_ingest_ms")),
AnalyzeMs = reader.GetInt32(reader.GetOrdinal("t_analyze_ms")),
ReachabilityMs = reader.GetInt32(reader.GetOrdinal("t_reachability_ms")),
VexMs = reader.GetInt32(reader.GetOrdinal("t_vex_ms")),
SignMs = reader.GetInt32(reader.GetOrdinal("t_sign_ms")),
PublishMs = reader.GetInt32(reader.GetOrdinal("t_publish_ms"))
},
PackageCount = reader.IsDBNull(reader.GetOrdinal("package_count"))
? null
: reader.GetInt32(reader.GetOrdinal("package_count")),
FindingCount = reader.IsDBNull(reader.GetOrdinal("finding_count"))
? null
: reader.GetInt32(reader.GetOrdinal("finding_count")),
VexDecisionCount = reader.IsDBNull(reader.GetOrdinal("vex_decision_count"))
? null
: reader.GetInt32(reader.GetOrdinal("vex_decision_count")),
ScannerVersion = reader.GetString(reader.GetOrdinal("scanner_version")),
ScannerImageDigest = reader.IsDBNull(reader.GetOrdinal("scanner_image_digest"))
? null
: reader.GetString(reader.GetOrdinal("scanner_image_digest")),
IsReplay = reader.GetBoolean(reader.GetOrdinal("is_replay")),
CreatedAt = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("created_at"))
};
}
private static ExecutionPhase MapToExecutionPhase(NpgsqlDataReader reader)
{
var phaseMetricsJson = reader.IsDBNull(reader.GetOrdinal("phase_metrics"))
? null
: reader.GetString(reader.GetOrdinal("phase_metrics"));
return new ExecutionPhase
{
Id = reader.GetInt64(reader.GetOrdinal("id")),
MetricsId = reader.GetGuid(reader.GetOrdinal("metrics_id")),
PhaseName = reader.GetString(reader.GetOrdinal("phase_name")),
PhaseOrder = reader.GetInt32(reader.GetOrdinal("phase_order")),
StartedAt = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("started_at")),
FinishedAt = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("finished_at")),
Success = reader.GetBoolean(reader.GetOrdinal("success")),
ErrorCode = reader.IsDBNull(reader.GetOrdinal("error_code"))
? null
: reader.GetString(reader.GetOrdinal("error_code")),
ErrorMessage = reader.IsDBNull(reader.GetOrdinal("error_message"))
? null
: reader.GetString(reader.GetOrdinal("error_message")),
PhaseMetrics = phaseMetricsJson is not null
? JsonSerializer.Deserialize<Dictionary<string, object>>(phaseMetricsJson)
: null
};
}
private static TteStats MapToTteStats(NpgsqlDataReader reader)
{
return new TteStats
{
TenantId = reader.GetGuid(reader.GetOrdinal("tenant_id")),
HourBucket = reader.GetFieldValue<DateTimeOffset>(reader.GetOrdinal("hour_bucket")),
ScanCount = reader.GetInt32(reader.GetOrdinal("scan_count")),
TteAvgMs = reader.GetInt32(reader.GetOrdinal("tte_avg_ms")),
TteP50Ms = reader.GetInt32(reader.GetOrdinal("tte_p50_ms")),
TteP95Ms = reader.GetInt32(reader.GetOrdinal("tte_p95_ms")),
TteMaxMs = reader.GetInt32(reader.GetOrdinal("tte_max_ms")),
SloP50CompliancePercent = reader.GetDecimal(reader.GetOrdinal("slo_p50_compliance_percent")),
SloP95CompliancePercent = reader.GetDecimal(reader.GetOrdinal("slo_p95_compliance_percent"))
};
}
}

192
src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Fixtures/ScaCatalogueDeterminismTests.cs Normal file
View File

@@ -0,0 +1,192 @@
// -----------------------------------------------------------------------------
// ScaCatalogueDeterminismTests.cs
// Sprint: SPRINT_0351_0001_0001_sca_failure_catalogue_completion
// Task: SCA-0351-010
// Description: Determinism validation for SCA Failure Catalogue fixtures
// -----------------------------------------------------------------------------
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
namespace StellaOps.Scanner.Core.Tests.Fixtures;
/// <summary>
/// Validates determinism properties of SCA Failure Catalogue fixtures.
/// These tests ensure that fixture content is:
/// 1. Content-addressable (hash-based identification)
/// 2. Reproducible (same content produces same hash)
/// 3. Tamper-evident (changes are detectable)
/// </summary>
public class ScaCatalogueDeterminismTests
{
private const string CatalogueBasePath = "../../../../../../tests/fixtures/sca/catalogue";
[Theory]
[InlineData("fc6")]
[InlineData("fc7")]
[InlineData("fc8")]
[InlineData("fc9")]
[InlineData("fc10")]
public void Fixture_HasStableContentHash(string fixtureId)
{
var fixturePath = Path.Combine(CatalogueBasePath, fixtureId);
if (!Directory.Exists(fixturePath)) return;
// Compute hash of all fixture files
var hash1 = ComputeFixtureHash(fixturePath);
var hash2 = ComputeFixtureHash(fixturePath);
Assert.Equal(hash1, hash2);
Assert.NotEmpty(hash1);
}
[Theory]
[InlineData("fc6")]
[InlineData("fc7")]
[InlineData("fc8")]
[InlineData("fc9")]
[InlineData("fc10")]
public void Fixture_ManifestHasRequiredFields(string fixtureId)
{
var manifestPath = Path.Combine(CatalogueBasePath, fixtureId, "manifest.json");
if (!File.Exists(manifestPath)) return;
var json = File.ReadAllText(manifestPath);
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
// Required fields for deterministic fixtures
Assert.True(root.TryGetProperty("id", out _), "manifest missing 'id'");
Assert.True(root.TryGetProperty("description", out _), "manifest missing 'description'");
Assert.True(root.TryGetProperty("failureMode", out _), "manifest missing 'failureMode'");
}
[Theory]
[InlineData("fc6")]
[InlineData("fc7")]
[InlineData("fc8")]
[InlineData("fc9")]
[InlineData("fc10")]
public void Fixture_NoExternalDependencies(string fixtureId)
{
var fixturePath = Path.Combine(CatalogueBasePath, fixtureId);
if (!Directory.Exists(fixturePath)) return;
var files = Directory.GetFiles(fixturePath, "*", SearchOption.AllDirectories);
foreach (var file in files)
{
var content = File.ReadAllText(file);
// Check for common external URL patterns that would break offline operation
Assert.DoesNotContain("http://", content.ToLowerInvariant().Replace("https://", ""));
// Allow https only for documentation references, not actual fetches
var httpsCount = CountOccurrences(content.ToLowerInvariant(), "https://");
if (httpsCount > 0)
{
// If HTTPS URLs exist, they should be in comments or documentation
// Real fixtures shouldn't require network access
var extension = Path.GetExtension(file).ToLowerInvariant();
if (extension is ".json" or ".yaml" or ".yml")
{
// For data files, URLs should only be in documentation fields
// This is a soft check - actual network isolation is tested elsewhere
}
}
}
}
[Theory]
[InlineData("fc6")]
[InlineData("fc7")]
[InlineData("fc8")]
[InlineData("fc9")]
[InlineData("fc10")]
public void Fixture_FilesAreSorted(string fixtureId)
{
var fixturePath = Path.Combine(CatalogueBasePath, fixtureId);
if (!Directory.Exists(fixturePath)) return;
// File ordering should be deterministic
var files1 = Directory.GetFiles(fixturePath, "*", SearchOption.AllDirectories)
.Select(f => Path.GetRelativePath(fixturePath, f))
.OrderBy(f => f, StringComparer.Ordinal)
.ToList();
var files2 = Directory.GetFiles(fixturePath, "*", SearchOption.AllDirectories)
.Select(f => Path.GetRelativePath(fixturePath, f))
.OrderBy(f => f, StringComparer.Ordinal)
.ToList();
Assert.Equal(files1, files2);
}
[Fact]
public void InputsLock_IsDeterministic()
{
var inputsLockPath = Path.Combine(CatalogueBasePath, "inputs.lock");
if (!File.Exists(inputsLockPath)) return;
// Compute hash twice
var bytes = File.ReadAllBytes(inputsLockPath);
var hash1 = SHA256.HashData(bytes);
var hash2 = SHA256.HashData(bytes);
Assert.Equal(hash1, hash2);
}
[Fact]
public void InputsLock_ContainsAllFixtures()
{
var inputsLockPath = Path.Combine(CatalogueBasePath, "inputs.lock");
if (!File.Exists(inputsLockPath)) return;
var content = File.ReadAllText(inputsLockPath);
// All FC6-FC10 fixtures should be referenced
Assert.Contains("fc6", content.ToLowerInvariant());
Assert.Contains("fc7", content.ToLowerInvariant());
Assert.Contains("fc8", content.ToLowerInvariant());
Assert.Contains("fc9", content.ToLowerInvariant());
Assert.Contains("fc10", content.ToLowerInvariant());
}
#region Helper Methods
private static string ComputeFixtureHash(string fixturePath)
{
var files = Directory.GetFiles(fixturePath, "*", SearchOption.AllDirectories)
.OrderBy(f => f, StringComparer.Ordinal)
.ToList();
using var sha256 = SHA256.Create();
var combined = new StringBuilder();
foreach (var file in files)
{
var relativePath = Path.GetRelativePath(fixturePath, file);
var fileBytes = File.ReadAllBytes(file);
var fileHash = Convert.ToHexStringLower(SHA256.HashData(fileBytes));
combined.AppendLine($"{relativePath}:{fileHash}");
}
var bytes = Encoding.UTF8.GetBytes(combined.ToString());
return Convert.ToHexStringLower(SHA256.HashData(bytes));
}
private static int CountOccurrences(string source, string pattern)
{
var count = 0;
var index = 0;
while ((index = source.IndexOf(pattern, index, StringComparison.Ordinal)) != -1)
{
count++;
index += pattern.Length;
}
return count;
}
#endregion
}

295
src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Fixtures/ScaFailureCatalogueTests.cs Normal file
View File

@@ -0,0 +1,295 @@
// -----------------------------------------------------------------------------
// ScaFailureCatalogueTests.cs
// Sprint: SPRINT_0351_0001_0001_sca_failure_catalogue_completion
// Task: SCA-0351-008
// Description: xUnit tests for SCA Failure Catalogue FC6-FC10
// -----------------------------------------------------------------------------
using System.Text.Json;
namespace StellaOps.Scanner.Core.Tests.Fixtures;
/// <summary>
/// Tests for SCA Failure Catalogue cases FC6-FC10.
/// Each test validates that the scanner correctly handles a specific real-world failure mode.
/// </summary>
/// <remarks>
/// Fixture directory: tests/fixtures/sca/catalogue/
///
/// FC6: Java Shadow JAR - Fat/uber JARs with shaded dependencies
/// FC7: .NET Transitive Pinning - Transitive dependency version conflicts
/// FC8: Docker Multi-Stage Leakage - Build-time dependencies in runtime
/// FC9: PURL Namespace Collision - Same package name in different ecosystems
/// FC10: CVE Split/Merge - Vulnerability split across multiple CVEs
/// </remarks>
public class ScaFailureCatalogueTests
{
private const string CatalogueBasePath = "../../../../../../tests/fixtures/sca/catalogue";
#region FC6: Java Shadow JAR
[Fact]
public void FC6_ShadowJar_ManifestExists()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc6", "manifest.json");
Assert.True(File.Exists(manifestPath), $"FC6 manifest not found at {manifestPath}");
}
[Fact]
public void FC6_ShadowJar_HasExpectedFiles()
{
var fc6Path = Path.Combine(CatalogueBasePath, "fc6");
Assert.True(Directory.Exists(fc6Path), "FC6 directory not found");
var files = Directory.GetFiles(fc6Path, "*", SearchOption.AllDirectories);
Assert.NotEmpty(files);
}
[Fact]
public void FC6_ShadowJar_ManifestIsValid()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc6", "manifest.json");
if (!File.Exists(manifestPath)) return; // Skip if not present
var json = File.ReadAllText(manifestPath);
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
Assert.NotNull(manifest);
Assert.Equal("FC6", manifest.Id);
Assert.NotEmpty(manifest.Description);
Assert.NotEmpty(manifest.ExpectedFindings);
}
#endregion
#region FC7: .NET Transitive Pinning
[Fact]
public void FC7_TransitivePinning_ManifestExists()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc7", "manifest.json");
Assert.True(File.Exists(manifestPath), $"FC7 manifest not found at {manifestPath}");
}
[Fact]
public void FC7_TransitivePinning_HasExpectedFiles()
{
var fc7Path = Path.Combine(CatalogueBasePath, "fc7");
Assert.True(Directory.Exists(fc7Path), "FC7 directory not found");
var files = Directory.GetFiles(fc7Path, "*", SearchOption.AllDirectories);
Assert.NotEmpty(files);
}
[Fact]
public void FC7_TransitivePinning_ManifestIsValid()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc7", "manifest.json");
if (!File.Exists(manifestPath)) return;
var json = File.ReadAllText(manifestPath);
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
Assert.NotNull(manifest);
Assert.Equal("FC7", manifest.Id);
Assert.NotEmpty(manifest.ExpectedFindings);
}
#endregion
#region FC8: Docker Multi-Stage Leakage
[Fact]
public void FC8_MultiStageLeakage_ManifestExists()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc8", "manifest.json");
Assert.True(File.Exists(manifestPath), $"FC8 manifest not found at {manifestPath}");
}
[Fact]
public void FC8_MultiStageLeakage_HasDockerfile()
{
var fc8Path = Path.Combine(CatalogueBasePath, "fc8");
Assert.True(Directory.Exists(fc8Path), "FC8 directory not found");
// Multi-stage leakage tests should have Dockerfile examples
var dockerfiles = Directory.GetFiles(fc8Path, "Dockerfile*", SearchOption.AllDirectories);
Assert.NotEmpty(dockerfiles);
}
[Fact]
public void FC8_MultiStageLeakage_ManifestIsValid()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc8", "manifest.json");
if (!File.Exists(manifestPath)) return;
var json = File.ReadAllText(manifestPath);
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
Assert.NotNull(manifest);
Assert.Equal("FC8", manifest.Id);
}
#endregion
#region FC9: PURL Namespace Collision
[Fact]
public void FC9_PurlNamespaceCollision_ManifestExists()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc9", "manifest.json");
Assert.True(File.Exists(manifestPath), $"FC9 manifest not found at {manifestPath}");
}
[Fact]
public void FC9_PurlNamespaceCollision_HasMultipleEcosystems()
{
var fc9Path = Path.Combine(CatalogueBasePath, "fc9");
Assert.True(Directory.Exists(fc9Path), "FC9 directory not found");
// Should contain files for multiple ecosystems
var files = Directory.GetFiles(fc9Path, "*", SearchOption.AllDirectories)
.Select(f => Path.GetFileName(f))
.ToList();
Assert.NotEmpty(files);
}
[Fact]
public void FC9_PurlNamespaceCollision_ManifestIsValid()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc9", "manifest.json");
if (!File.Exists(manifestPath)) return;
var json = File.ReadAllText(manifestPath);
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
Assert.NotNull(manifest);
Assert.Equal("FC9", manifest.Id);
}
#endregion
#region FC10: CVE Split/Merge
[Fact]
public void FC10_CveSplitMerge_ManifestExists()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc10", "manifest.json");
Assert.True(File.Exists(manifestPath), $"FC10 manifest not found at {manifestPath}");
}
[Fact]
public void FC10_CveSplitMerge_ManifestIsValid()
{
var manifestPath = Path.Combine(CatalogueBasePath, "fc10", "manifest.json");
if (!File.Exists(manifestPath)) return;
var json = File.ReadAllText(manifestPath);
var manifest = JsonSerializer.Deserialize<CatalogueManifest>(json);
Assert.NotNull(manifest);
Assert.Equal("FC10", manifest.Id);
// CVE split/merge should have multiple related CVEs
Assert.NotNull(manifest.RelatedCves);
Assert.True(manifest.RelatedCves.Count >= 2, "CVE split/merge should have at least 2 related CVEs");
}
#endregion
#region Cross-Catalogue Tests
[Fact]
public void AllCatalogueFixtures_HaveInputsLock()
{
var inputsLockPath = Path.Combine(CatalogueBasePath, "inputs.lock");
Assert.True(File.Exists(inputsLockPath), "inputs.lock not found");
var content = File.ReadAllText(inputsLockPath);
Assert.NotEmpty(content);
}
[Theory]
[InlineData("fc6")]
[InlineData("fc7")]
[InlineData("fc8")]
[InlineData("fc9")]
[InlineData("fc10")]
public void CatalogueFixture_DirectoryExists(string fixtureId)
{
var fixturePath = Path.Combine(CatalogueBasePath, fixtureId);
Assert.True(Directory.Exists(fixturePath), $"Fixture {fixtureId} directory not found");
}
[Theory]
[InlineData("fc6")]
[InlineData("fc7")]
[InlineData("fc8")]
[InlineData("fc9")]
[InlineData("fc10")]
public void CatalogueFixture_HasManifest(string fixtureId)
{
var manifestPath = Path.Combine(CatalogueBasePath, fixtureId, "manifest.json");
Assert.True(File.Exists(manifestPath), $"Fixture {fixtureId} manifest not found");
}
#endregion
#region Determinism Tests
[Theory]
[InlineData("fc6")]
[InlineData("fc7")]
[InlineData("fc8")]
[InlineData("fc9")]
[InlineData("fc10")]
public void CatalogueFixture_ManifestIsDeterministic(string fixtureId)
{
var manifestPath = Path.Combine(CatalogueBasePath, fixtureId, "manifest.json");
if (!File.Exists(manifestPath)) return;
// Read twice and ensure identical
var content1 = File.ReadAllText(manifestPath);
var content2 = File.ReadAllText(manifestPath);
Assert.Equal(content1, content2);
// Verify can be parsed to consistent structure
var manifest1 = JsonSerializer.Deserialize<CatalogueManifest>(content1);
var manifest2 = JsonSerializer.Deserialize<CatalogueManifest>(content2);
Assert.NotNull(manifest1);
Assert.NotNull(manifest2);
Assert.Equal(manifest1.Id, manifest2.Id);
Assert.Equal(manifest1.Description, manifest2.Description);
}
#endregion
#region Test Models
private record CatalogueManifest
{
public string Id { get; init; } = "";
public string Description { get; init; } = "";
public string FailureMode { get; init; } = "";
public List<ExpectedFinding> ExpectedFindings { get; init; } = [];
public List<string> RelatedCves { get; init; } = [];
public DsseManifest? Dsse { get; init; }
}
private record ExpectedFinding
{
public string Purl { get; init; } = "";
public string VulnerabilityId { get; init; } = "";
public string ExpectedResult { get; init; } = "";
}
private record DsseManifest
{
public string PayloadType { get; init; } = "";
public string Signature { get; init; } = "";
}
#endregion
}