stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
Export Center CI / export-ci (push) Has been cancelled
Details
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Details
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Details
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Details
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-13 00:20:26 +02:00
parent e1f1bef4c1
commit 564df71bfb
2376 changed files with 334389 additions and 328032 deletions
Download Patch File Download Diff File Expand all files Collapse all files

498
src/Bench/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PolicyScenarioRunner.cs
View File

@@ -1,249 +1,249 @@
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using StellaOps.Policy;
namespace StellaOps.Bench.PolicyEngine;
internal sealed class PolicyScenarioRunner
{
private readonly PolicyScenarioConfig _config;
private readonly PolicyDocument _document;
private readonly PolicyScoringConfig _scoringConfig;
private readonly PolicyFinding[] _findings;
public PolicyScenarioRunner(PolicyScenarioConfig config, string repoRoot)
{
_config = config ?? throw new ArgumentNullException(nameof(config));
ArgumentException.ThrowIfNullOrWhiteSpace(repoRoot);
var policyPath = ResolvePathWithinRoot(repoRoot, config.PolicyPath);
var policyContent = File.ReadAllText(policyPath);
var policyFormat = PolicySchema.DetectFormat(policyPath);
var binding = PolicyBinder.Bind(policyContent, policyFormat);
if (!binding.Success)
{
var issues = string.Join(", ", binding.Issues.Select(issue => issue.Code));
throw new InvalidOperationException($"Policy '{config.PolicyPath}' failed validation: {issues}.");
}
_document = binding.Document;
_scoringConfig = LoadScoringConfig(repoRoot, config.ScoringConfigPath);
_findings = SyntheticFindingGenerator.Create(config, repoRoot);
}
public ScenarioExecutionResult Execute(int iterations, CancellationToken cancellationToken)
{
if (iterations <= 0)
{
throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive.");
}
var durations = new double[iterations];
var throughputs = new double[iterations];
var allocations = new double[iterations];
var hashingAccumulator = new EvaluationAccumulator();
for (var index = 0; index < iterations; index++)
{
cancellationToken.ThrowIfCancellationRequested();
var beforeAllocated = GC.GetTotalAllocatedBytes();
var stopwatch = Stopwatch.StartNew();
hashingAccumulator.Reset();
foreach (var finding in _findings)
{
var verdict = PolicyEvaluation.EvaluateFinding(_document, _scoringConfig, finding);
hashingAccumulator.Add(verdict);
}
stopwatch.Stop();
var afterAllocated = GC.GetTotalAllocatedBytes();
var elapsedMs = stopwatch.Elapsed.TotalMilliseconds;
if (elapsedMs <= 0)
{
elapsedMs = 0.0001;
}
durations[index] = elapsedMs;
throughputs[index] = _findings.Length / stopwatch.Elapsed.TotalSeconds;
allocations[index] = Math.Max(0, afterAllocated - beforeAllocated) / (1024d * 1024d);
hashingAccumulator.AssertConsumed();
}
return new ScenarioExecutionResult(
durations,
throughputs,
allocations,
_findings.Length);
}
private static PolicyScoringConfig LoadScoringConfig(string repoRoot, string? scoringPath)
{
if (string.IsNullOrWhiteSpace(scoringPath))
{
return PolicyScoringConfig.Default;
}
var resolved = ResolvePathWithinRoot(repoRoot, scoringPath);
var format = PolicySchema.DetectFormat(resolved);
var content = File.ReadAllText(resolved);
var binding = PolicyScoringConfigBinder.Bind(content, format);
if (!binding.Success || binding.Config is null)
{
var issues = binding.Issues.Length == 0
? "unknown"
: string.Join(", ", binding.Issues.Select(issue => issue.Code));
throw new InvalidOperationException($"Scoring configuration '{scoringPath}' failed validation: {issues}.");
}
return binding.Config;
}
private static string ResolvePathWithinRoot(string repoRoot, string relativePath)
{
ArgumentException.ThrowIfNullOrWhiteSpace(repoRoot);
ArgumentException.ThrowIfNullOrWhiteSpace(relativePath);
var combined = Path.GetFullPath(Path.Combine(repoRoot, relativePath));
if (!PathUtilities.IsWithinRoot(repoRoot, combined))
{
throw new InvalidOperationException($"Path '{relativePath}' escapes repository root '{repoRoot}'.");
}
if (!File.Exists(combined))
{
throw new FileNotFoundException($"Path '{relativePath}' resolved to '{combined}' but does not exist.", combined);
}
return combined;
}
}
internal sealed record ScenarioExecutionResult(
IReadOnlyList<double> Durations,
IReadOnlyList<double> Throughputs,
IReadOnlyList<double> AllocatedMb,
int FindingCount);
internal static class SyntheticFindingGenerator
{
private static readonly ImmutableArray<string> Environments = ImmutableArray.Create("prod", "staging", "dev");
private static readonly ImmutableArray<string> Sources = ImmutableArray.Create("concelier", "excitor", "sbom");
private static readonly ImmutableArray<string> Vendors = ImmutableArray.Create("acme", "contoso", "globex", "initech", "umbrella");
private static readonly ImmutableArray<string> Licenses = ImmutableArray.Create("MIT", "Apache-2.0", "GPL-3.0", "BSD-3-Clause", "Proprietary");
private static readonly ImmutableArray<string> Repositories = ImmutableArray.Create("acme/service-api", "acme/web", "acme/worker", "acme/mobile", "acme/cli");
private static readonly ImmutableArray<string> Images = ImmutableArray.Create("registry.local/worker:2025.10", "registry.local/api:2025.10", "registry.local/cli:2025.10");
private static readonly ImmutableArray<string> TagPool = ImmutableArray.Create("kev", "runtime", "reachable", "public", "third-party", "critical-path");
private static readonly ImmutableArray<ImmutableArray<string>> TagSets = BuildTagSets();
private static readonly PolicySeverity[] SeverityPool =
{
PolicySeverity.Critical,
PolicySeverity.High,
PolicySeverity.Medium,
PolicySeverity.Low,
PolicySeverity.Informational
};
public static PolicyFinding[] Create(PolicyScenarioConfig config, string repoRoot)
{
var totalFindings = config.ResolveFindingCount();
if (totalFindings <= 0)
{
return Array.Empty<PolicyFinding>();
}
var seed = config.ResolveSeed();
var random = new Random(seed);
var findings = new PolicyFinding[totalFindings];
var tagsBuffer = new List<string>(3);
var componentCount = Math.Max(1, config.ComponentCount);
for (var index = 0; index < totalFindings; index++)
{
var componentIndex = index % componentCount;
var findingId = $"F-{componentIndex:D5}-{index:D6}";
var severity = SeverityPool[random.Next(SeverityPool.Length)];
var environment = Environments[componentIndex % Environments.Length];
var source = Sources[random.Next(Sources.Length)];
var vendor = Vendors[random.Next(Vendors.Length)];
var license = Licenses[random.Next(Licenses.Length)];
var repository = Repositories[componentIndex % Repositories.Length];
var image = Images[(componentIndex + index) % Images.Length];
var packageName = $"pkg{componentIndex % 1000}";
var purl = $"pkg:generic/{packageName}@{1 + (index % 20)}.{1 + (componentIndex % 10)}.{index % 5}";
var cve = index % 7 == 0 ? $"CVE-2025-{1000 + index % 9000:D4}" : null;
var layerDigest = $"sha256:{Convert.ToHexString(Guid.NewGuid().ToByteArray())[..32].ToLowerInvariant()}";
var tags = TagSets[random.Next(TagSets.Length)];
findings[index] = PolicyFinding.Create(
findingId,
severity,
environment: environment,
source: source,
vendor: vendor,
license: license,
image: image,
repository: repository,
package: packageName,
purl: purl,
cve: cve,
path: $"/app/{packageName}/{index % 50}.so",
layerDigest: layerDigest,
tags: tags);
}
return findings;
}
private static ImmutableArray<ImmutableArray<string>> BuildTagSets()
{
var builder = ImmutableArray.CreateBuilder<ImmutableArray<string>>();
builder.Add(ImmutableArray<string>.Empty);
builder.Add(ImmutableArray.Create("kev"));
builder.Add(ImmutableArray.Create("runtime"));
builder.Add(ImmutableArray.Create("reachable"));
builder.Add(ImmutableArray.Create("third-party"));
builder.Add(ImmutableArray.Create("kev", "runtime"));
builder.Add(ImmutableArray.Create("kev", "third-party"));
builder.Add(ImmutableArray.Create("runtime", "public"));
builder.Add(ImmutableArray.Create("reachable", "critical-path"));
return builder.ToImmutable();
}
}
internal sealed class EvaluationAccumulator
{
private double _scoreAccumulator;
private int _quietCount;
public void Reset()
{
_scoreAccumulator = 0;
_quietCount = 0;
}
public void Add(PolicyVerdict verdict)
{
_scoreAccumulator += verdict.Score;
if (verdict.Quiet)
{
_quietCount++;
}
}
public void AssertConsumed()
{
if (_scoreAccumulator == 0 && _quietCount == 0)
{
throw new InvalidOperationException("Evaluation accumulator detected zero work; dataset may be empty.");
}
}
}
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using StellaOps.Policy;
namespace StellaOps.Bench.PolicyEngine;
internal sealed class PolicyScenarioRunner
{
private readonly PolicyScenarioConfig _config;
private readonly PolicyDocument _document;
private readonly PolicyScoringConfig _scoringConfig;
private readonly PolicyFinding[] _findings;
public PolicyScenarioRunner(PolicyScenarioConfig config, string repoRoot)
{
_config = config ?? throw new ArgumentNullException(nameof(config));
ArgumentException.ThrowIfNullOrWhiteSpace(repoRoot);
var policyPath = ResolvePathWithinRoot(repoRoot, config.PolicyPath);
var policyContent = File.ReadAllText(policyPath);
var policyFormat = PolicySchema.DetectFormat(policyPath);
var binding = PolicyBinder.Bind(policyContent, policyFormat);
if (!binding.Success)
{
var issues = string.Join(", ", binding.Issues.Select(issue => issue.Code));
throw new InvalidOperationException($"Policy '{config.PolicyPath}' failed validation: {issues}.");
}
_document = binding.Document;
_scoringConfig = LoadScoringConfig(repoRoot, config.ScoringConfigPath);
_findings = SyntheticFindingGenerator.Create(config, repoRoot);
}
public ScenarioExecutionResult Execute(int iterations, CancellationToken cancellationToken)
{
if (iterations <= 0)
{
throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive.");
}
var durations = new double[iterations];
var throughputs = new double[iterations];
var allocations = new double[iterations];
var hashingAccumulator = new EvaluationAccumulator();
for (var index = 0; index < iterations; index++)
{
cancellationToken.ThrowIfCancellationRequested();
var beforeAllocated = GC.GetTotalAllocatedBytes();
var stopwatch = Stopwatch.StartNew();
hashingAccumulator.Reset();
foreach (var finding in _findings)
{
var verdict = PolicyEvaluation.EvaluateFinding(_document, _scoringConfig, finding);
hashingAccumulator.Add(verdict);
}
stopwatch.Stop();
var afterAllocated = GC.GetTotalAllocatedBytes();
var elapsedMs = stopwatch.Elapsed.TotalMilliseconds;
if (elapsedMs <= 0)
{
elapsedMs = 0.0001;
}
durations[index] = elapsedMs;
throughputs[index] = _findings.Length / stopwatch.Elapsed.TotalSeconds;
allocations[index] = Math.Max(0, afterAllocated - beforeAllocated) / (1024d * 1024d);
hashingAccumulator.AssertConsumed();
}
return new ScenarioExecutionResult(
durations,
throughputs,
allocations,
_findings.Length);
}
private static PolicyScoringConfig LoadScoringConfig(string repoRoot, string? scoringPath)
{
if (string.IsNullOrWhiteSpace(scoringPath))
{
return PolicyScoringConfig.Default;
}
var resolved = ResolvePathWithinRoot(repoRoot, scoringPath);
var format = PolicySchema.DetectFormat(resolved);
var content = File.ReadAllText(resolved);
var binding = PolicyScoringConfigBinder.Bind(content, format);
if (!binding.Success || binding.Config is null)
{
var issues = binding.Issues.Length == 0
? "unknown"
: string.Join(", ", binding.Issues.Select(issue => issue.Code));
throw new InvalidOperationException($"Scoring configuration '{scoringPath}' failed validation: {issues}.");
}
return binding.Config;
}
private static string ResolvePathWithinRoot(string repoRoot, string relativePath)
{
ArgumentException.ThrowIfNullOrWhiteSpace(repoRoot);
ArgumentException.ThrowIfNullOrWhiteSpace(relativePath);
var combined = Path.GetFullPath(Path.Combine(repoRoot, relativePath));
if (!PathUtilities.IsWithinRoot(repoRoot, combined))
{
throw new InvalidOperationException($"Path '{relativePath}' escapes repository root '{repoRoot}'.");
}
if (!File.Exists(combined))
{
throw new FileNotFoundException($"Path '{relativePath}' resolved to '{combined}' but does not exist.", combined);
}
return combined;
}
}
internal sealed record ScenarioExecutionResult(
IReadOnlyList<double> Durations,
IReadOnlyList<double> Throughputs,
IReadOnlyList<double> AllocatedMb,
int FindingCount);
internal static class SyntheticFindingGenerator
{
private static readonly ImmutableArray<string> Environments = ImmutableArray.Create("prod", "staging", "dev");
private static readonly ImmutableArray<string> Sources = ImmutableArray.Create("concelier", "excitor", "sbom");
private static readonly ImmutableArray<string> Vendors = ImmutableArray.Create("acme", "contoso", "globex", "initech", "umbrella");
private static readonly ImmutableArray<string> Licenses = ImmutableArray.Create("MIT", "Apache-2.0", "GPL-3.0", "BSD-3-Clause", "Proprietary");
private static readonly ImmutableArray<string> Repositories = ImmutableArray.Create("acme/service-api", "acme/web", "acme/worker", "acme/mobile", "acme/cli");
private static readonly ImmutableArray<string> Images = ImmutableArray.Create("registry.local/worker:2025.10", "registry.local/api:2025.10", "registry.local/cli:2025.10");
private static readonly ImmutableArray<string> TagPool = ImmutableArray.Create("kev", "runtime", "reachable", "public", "third-party", "critical-path");
private static readonly ImmutableArray<ImmutableArray<string>> TagSets = BuildTagSets();
private static readonly PolicySeverity[] SeverityPool =
{
PolicySeverity.Critical,
PolicySeverity.High,
PolicySeverity.Medium,
PolicySeverity.Low,
PolicySeverity.Informational
};
public static PolicyFinding[] Create(PolicyScenarioConfig config, string repoRoot)
{
var totalFindings = config.ResolveFindingCount();
if (totalFindings <= 0)
{
return Array.Empty<PolicyFinding>();
}
var seed = config.ResolveSeed();
var random = new Random(seed);
var findings = new PolicyFinding[totalFindings];
var tagsBuffer = new List<string>(3);
var componentCount = Math.Max(1, config.ComponentCount);
for (var index = 0; index < totalFindings; index++)
{
var componentIndex = index % componentCount;
var findingId = $"F-{componentIndex:D5}-{index:D6}";
var severity = SeverityPool[random.Next(SeverityPool.Length)];
var environment = Environments[componentIndex % Environments.Length];
var source = Sources[random.Next(Sources.Length)];
var vendor = Vendors[random.Next(Vendors.Length)];
var license = Licenses[random.Next(Licenses.Length)];
var repository = Repositories[componentIndex % Repositories.Length];
var image = Images[(componentIndex + index) % Images.Length];
var packageName = $"pkg{componentIndex % 1000}";
var purl = $"pkg:generic/{packageName}@{1 + (index % 20)}.{1 + (componentIndex % 10)}.{index % 5}";
var cve = index % 7 == 0 ? $"CVE-2025-{1000 + index % 9000:D4}" : null;
var layerDigest = $"sha256:{Convert.ToHexString(Guid.NewGuid().ToByteArray())[..32].ToLowerInvariant()}";
var tags = TagSets[random.Next(TagSets.Length)];
findings[index] = PolicyFinding.Create(
findingId,
severity,
environment: environment,
source: source,
vendor: vendor,
license: license,
image: image,
repository: repository,
package: packageName,
purl: purl,
cve: cve,
path: $"/app/{packageName}/{index % 50}.so",
layerDigest: layerDigest,
tags: tags);
}
return findings;
}
private static ImmutableArray<ImmutableArray<string>> BuildTagSets()
{
var builder = ImmutableArray.CreateBuilder<ImmutableArray<string>>();
builder.Add(ImmutableArray<string>.Empty);
builder.Add(ImmutableArray.Create("kev"));
builder.Add(ImmutableArray.Create("runtime"));
builder.Add(ImmutableArray.Create("reachable"));
builder.Add(ImmutableArray.Create("third-party"));
builder.Add(ImmutableArray.Create("kev", "runtime"));
builder.Add(ImmutableArray.Create("kev", "third-party"));
builder.Add(ImmutableArray.Create("runtime", "public"));
builder.Add(ImmutableArray.Create("reachable", "critical-path"));
return builder.ToImmutable();
}
}
internal sealed class EvaluationAccumulator
{
private double _scoreAccumulator;
private int _quietCount;
public void Reset()
{
_scoreAccumulator = 0;
_quietCount = 0;
}
public void Add(PolicyVerdict verdict)
{
_scoreAccumulator += verdict.Score;
if (verdict.Quiet)
{
_quietCount++;
}
}
public void AssertConsumed()
{
if (_scoreAccumulator == 0 && _quietCount == 0)
{
throw new InvalidOperationException("Evaluation accumulator detected zero work; dataset may be empty.");
}
}
}