using System.Collections.Immutable; using System.Diagnostics; using System.Globalization; using System.Linq; using StellaOps.Policy; namespace StellaOps.Bench.PolicyEngine; internal sealed class PolicyScenarioRunner { private readonly PolicyScenarioConfig _config; private readonly PolicyDocument _document; private readonly PolicyScoringConfig _scoringConfig; private readonly PolicyFinding[] _findings; public PolicyScenarioRunner(PolicyScenarioConfig config, string repoRoot) { _config = config ?? throw new ArgumentNullException(nameof(config)); ArgumentException.ThrowIfNullOrWhiteSpace(repoRoot); var policyPath = ResolvePathWithinRoot(repoRoot, config.PolicyPath); var policyContent = File.ReadAllText(policyPath); var policyFormat = PolicySchema.DetectFormat(policyPath); var binding = PolicyBinder.Bind(policyContent, policyFormat); if (!binding.Success) { var issues = string.Join(", ", binding.Issues.Select(issue => issue.Code)); throw new InvalidOperationException($"Policy '{config.PolicyPath}' failed validation: {issues}."); } _document = binding.Document; _scoringConfig = LoadScoringConfig(repoRoot, config.ScoringConfigPath); _findings = SyntheticFindingGenerator.Create(config, repoRoot); } public ScenarioExecutionResult Execute(int iterations, CancellationToken cancellationToken) { if (iterations <= 0) { throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive."); } var durations = new double[iterations]; var throughputs = new double[iterations]; var allocations = new double[iterations]; var hashingAccumulator = new EvaluationAccumulator(); for (var index = 0; index < iterations; index++) { cancellationToken.ThrowIfCancellationRequested(); var beforeAllocated = GC.GetTotalAllocatedBytes(); var stopwatch = Stopwatch.StartNew(); hashingAccumulator.Reset(); foreach (var finding in _findings) { var verdict = PolicyEvaluation.EvaluateFinding(_document, _scoringConfig, finding); hashingAccumulator.Add(verdict); } stopwatch.Stop(); var afterAllocated = GC.GetTotalAllocatedBytes(); var elapsedMs = stopwatch.Elapsed.TotalMilliseconds; if (elapsedMs <= 0) { elapsedMs = 0.0001; } durations[index] = elapsedMs; throughputs[index] = _findings.Length / stopwatch.Elapsed.TotalSeconds; allocations[index] = Math.Max(0, afterAllocated - beforeAllocated) / (1024d * 1024d); hashingAccumulator.AssertConsumed(); } return new ScenarioExecutionResult( durations, throughputs, allocations, _findings.Length); } private static PolicyScoringConfig LoadScoringConfig(string repoRoot, string? scoringPath) { if (string.IsNullOrWhiteSpace(scoringPath)) { return PolicyScoringConfig.Default; } var resolved = ResolvePathWithinRoot(repoRoot, scoringPath); var format = PolicySchema.DetectFormat(resolved); var content = File.ReadAllText(resolved); var binding = PolicyScoringConfigBinder.Bind(content, format); if (!binding.Success || binding.Config is null) { var issues = binding.Issues.Length == 0 ? "unknown" : string.Join(", ", binding.Issues.Select(issue => issue.Code)); throw new InvalidOperationException($"Scoring configuration '{scoringPath}' failed validation: {issues}."); } return binding.Config; } private static string ResolvePathWithinRoot(string repoRoot, string relativePath) { ArgumentException.ThrowIfNullOrWhiteSpace(repoRoot); ArgumentException.ThrowIfNullOrWhiteSpace(relativePath); var combined = Path.GetFullPath(Path.Combine(repoRoot, relativePath)); if (!PathUtilities.IsWithinRoot(repoRoot, combined)) { throw new InvalidOperationException($"Path '{relativePath}' escapes repository root '{repoRoot}'."); } if (!File.Exists(combined)) { throw new FileNotFoundException($"Path '{relativePath}' resolved to '{combined}' but does not exist.", combined); } return combined; } } internal sealed record ScenarioExecutionResult( IReadOnlyList Durations, IReadOnlyList Throughputs, IReadOnlyList AllocatedMb, int FindingCount); internal static class SyntheticFindingGenerator { private static readonly ImmutableArray Environments = ImmutableArray.Create("prod", "staging", "dev"); private static readonly ImmutableArray Sources = ImmutableArray.Create("concelier", "excitor", "sbom"); private static readonly ImmutableArray Vendors = ImmutableArray.Create("acme", "contoso", "globex", "initech", "umbrella"); private static readonly ImmutableArray Licenses = ImmutableArray.Create("MIT", "Apache-2.0", "GPL-3.0", "BSD-3-Clause", "Proprietary"); private static readonly ImmutableArray Repositories = ImmutableArray.Create("acme/service-api", "acme/web", "acme/worker", "acme/mobile", "acme/cli"); private static readonly ImmutableArray Images = ImmutableArray.Create("registry.local/worker:2025.10", "registry.local/api:2025.10", "registry.local/cli:2025.10"); private static readonly ImmutableArray TagPool = ImmutableArray.Create("kev", "runtime", "reachable", "public", "third-party", "critical-path"); private static readonly ImmutableArray> TagSets = BuildTagSets(); private static readonly PolicySeverity[] SeverityPool = { PolicySeverity.Critical, PolicySeverity.High, PolicySeverity.Medium, PolicySeverity.Low, PolicySeverity.Informational }; public static PolicyFinding[] Create(PolicyScenarioConfig config, string repoRoot) { var totalFindings = config.ResolveFindingCount(); if (totalFindings <= 0) { return Array.Empty(); } var seed = config.ResolveSeed(); var random = new Random(seed); var findings = new PolicyFinding[totalFindings]; var tagsBuffer = new List(3); var componentCount = Math.Max(1, config.ComponentCount); for (var index = 0; index < totalFindings; index++) { var componentIndex = index % componentCount; var findingId = $"F-{componentIndex:D5}-{index:D6}"; var severity = SeverityPool[random.Next(SeverityPool.Length)]; var environment = Environments[componentIndex % Environments.Length]; var source = Sources[random.Next(Sources.Length)]; var vendor = Vendors[random.Next(Vendors.Length)]; var license = Licenses[random.Next(Licenses.Length)]; var repository = Repositories[componentIndex % Repositories.Length]; var image = Images[(componentIndex + index) % Images.Length]; var packageName = $"pkg{componentIndex % 1000}"; var purl = $"pkg:generic/{packageName}@{1 + (index % 20)}.{1 + (componentIndex % 10)}.{index % 5}"; var cve = index % 7 == 0 ? $"CVE-2025-{1000 + index % 9000:D4}" : null; var layerDigest = $"sha256:{Convert.ToHexString(Guid.NewGuid().ToByteArray())[..32].ToLowerInvariant()}"; var tags = TagSets[random.Next(TagSets.Length)]; findings[index] = PolicyFinding.Create( findingId, severity, environment: environment, source: source, vendor: vendor, license: license, image: image, repository: repository, package: packageName, purl: purl, cve: cve, path: $"/app/{packageName}/{index % 50}.so", layerDigest: layerDigest, tags: tags); } return findings; } private static ImmutableArray> BuildTagSets() { var builder = ImmutableArray.CreateBuilder>(); builder.Add(ImmutableArray.Empty); builder.Add(ImmutableArray.Create("kev")); builder.Add(ImmutableArray.Create("runtime")); builder.Add(ImmutableArray.Create("reachable")); builder.Add(ImmutableArray.Create("third-party")); builder.Add(ImmutableArray.Create("kev", "runtime")); builder.Add(ImmutableArray.Create("kev", "third-party")); builder.Add(ImmutableArray.Create("runtime", "public")); builder.Add(ImmutableArray.Create("reachable", "critical-path")); return builder.ToImmutable(); } } internal sealed class EvaluationAccumulator { private double _scoreAccumulator; private int _quietCount; public void Reset() { _scoreAccumulator = 0; _quietCount = 0; } public void Add(PolicyVerdict verdict) { _scoreAccumulator += verdict.Score; if (verdict.Quiet) { _quietCount++; } } public void AssertConsumed() { if (_scoreAccumulator == 0 && _quietCount == 0) { throw new InvalidOperationException("Evaluation accumulator detected zero work; dataset may be empty."); } } }