// SPDX-License-Identifier: BUSL-1.1 // Copyright © 2025 StellaOps // Sprint: SPRINT_8200_0012_0003_policy_engine_integration // Task: PINT-8200-044 - Benchmark: policy evaluation < 50ms per finding using System.Diagnostics; using FluentAssertions; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.Policy.Engine.Scoring.EvidenceWeightedScore; using StellaOps.Signals.EvidenceWeightedScore; using StellaOps.Signals.EvidenceWeightedScore.Normalizers; using Xunit; namespace StellaOps.Policy.Engine.Tests.Benchmarks; /// /// Benchmark tests verifying that EWS calculation meets performance requirements. /// Target: policy evaluation < 50ms per finding. /// [Trait("Category", "Benchmark")] [Trait("Category", "Performance")] [Trait("Sprint", "8200.0012.0003")] [Trait("Task", "PINT-8200-044")] public sealed class EwsCalculationBenchmarkTests { private const int TargetMaxMs = 50; private const int WarmupIterations = 100; private const int BenchmarkIterations = 1000; private static ServiceCollection CreateServicesWithConfiguration() { var services = new ServiceCollection(); var configuration = new ConfigurationBuilder() .AddInMemoryCollection() .Build(); services.AddSingleton(configuration); return services; } #region Calculator Performance Tests [Fact(DisplayName = "Single EWS calculation completes under 50ms")] public void SingleCalculation_CompletesUnder50ms() { // Arrange var calculator = new EvidenceWeightedScoreCalculator(); var input = CreateTestInput("perf-single"); // Warmup for (var i = 0; i < WarmupIterations; i++) { calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); } // Act var sw = Stopwatch.StartNew(); var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); sw.Stop(); // Assert result.Should().NotBeNull(); sw.ElapsedMilliseconds.Should().BeLessThan(TargetMaxMs, $"single EWS calculation should complete in under {TargetMaxMs}ms (actual: {sw.ElapsedMilliseconds}ms)"); } [Fact(DisplayName = "Average calculation time over 1000 iterations is under 1ms")] public void AverageCalculationTime_IsUnder1ms() { // Arrange var calculator = new EvidenceWeightedScoreCalculator(); var input = CreateTestInput("perf-avg"); // Warmup for (var i = 0; i < WarmupIterations; i++) { calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); } // Act var sw = Stopwatch.StartNew(); for (var i = 0; i < BenchmarkIterations; i++) { calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); } sw.Stop(); var avgMs = (double)sw.ElapsedMilliseconds / BenchmarkIterations; // Assert - average should be well under 1ms avgMs.Should().BeLessThan(1.0, $"average EWS calculation should be under 1ms (actual: {avgMs:F3}ms per calculation)"); } [Fact(DisplayName = "P99 calculation time is under 10ms")] public void P99CalculationTime_IsUnder10ms() { // Arrange var calculator = new EvidenceWeightedScoreCalculator(); var input = CreateTestInput("perf-p99"); var timings = new long[BenchmarkIterations]; // Warmup for (var i = 0; i < WarmupIterations; i++) { calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); } // Act - Collect timing for each iteration var sw = new Stopwatch(); for (var i = 0; i < BenchmarkIterations; i++) { sw.Restart(); calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); sw.Stop(); timings[i] = sw.ElapsedTicks; } // Calculate P99 Array.Sort(timings); var p99Index = (int)(BenchmarkIterations * 0.99); var p99Ticks = timings[p99Index]; var p99Ms = (double)p99Ticks / Stopwatch.Frequency * 1000; // Assert p99Ms.Should().BeLessThan(10.0, $"P99 EWS calculation time should be under 10ms (actual: {p99Ms:F3}ms)"); } #endregion #region Enricher Pipeline Performance Tests [Fact(DisplayName = "Single enrichment completes under 50ms")] public void SingleEnrichment_CompletesUnder50ms() { // Arrange var services = CreateServicesWithConfiguration(); services.AddEvidenceWeightedScoring(); services.AddEvidenceNormalizers(); services.AddEvidenceWeightedScore(opts => { opts.Enabled = true; opts.EnableCaching = false; // Measure actual calculation }); var provider = services.BuildServiceProvider(); var enricher = provider.GetRequiredService(); var evidence = CreateTestEvidence("perf-enricher"); // Warmup for (var i = 0; i < WarmupIterations; i++) { enricher.Enrich(evidence); } // Act var sw = Stopwatch.StartNew(); var result = enricher.Enrich(evidence); sw.Stop(); // Assert result.Should().NotBeNull(); sw.ElapsedMilliseconds.Should().BeLessThan(TargetMaxMs, $"single enrichment should complete in under {TargetMaxMs}ms (actual: {sw.ElapsedMilliseconds}ms)"); } [Fact(DisplayName = "Enricher with caching improves performance")] public void EnricherWithCaching_ImprovesPerformance() { // Arrange var services = CreateServicesWithConfiguration(); services.AddEvidenceWeightedScoring(); services.AddEvidenceNormalizers(); services.AddEvidenceWeightedScore(opts => { opts.Enabled = true; opts.EnableCaching = true; }); var provider = services.BuildServiceProvider(); var enricher = provider.GetRequiredService(); var evidence = CreateTestEvidence("perf-cache"); // Warmup and populate cache enricher.Enrich(evidence); // Act - Measure cached access var sw = Stopwatch.StartNew(); for (var i = 0; i < 100; i++) { enricher.Enrich(evidence); } sw.Stop(); var avgCachedMs = (double)sw.ElapsedMilliseconds / 100; // Assert - cached access should be very fast avgCachedMs.Should().BeLessThan(0.5, $"cached enrichment should be under 0.5ms (actual: {avgCachedMs:F3}ms)"); } #endregion #region Batch Performance Tests [Fact(DisplayName = "Batch of 100 findings processes under 500ms")] public void BatchOf100Findings_ProcessesUnder500ms() { // Arrange var calculator = new EvidenceWeightedScoreCalculator(); var inputs = Enumerable.Range(0, 100) .Select(i => CreateTestInput($"batch-{i}")) .ToList(); // Warmup foreach (var input in inputs.Take(10)) { calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); } // Act var sw = Stopwatch.StartNew(); foreach (var input in inputs) { calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); } sw.Stop(); // Assert - 100 findings * 50ms max = 5000ms, but should be much faster sw.ElapsedMilliseconds.Should().BeLessThan(500, $"batch of 100 findings should process in under 500ms (actual: {sw.ElapsedMilliseconds}ms)"); } [Fact(DisplayName = "Throughput exceeds 1000 evaluations per second")] public void Throughput_Exceeds1000PerSecond() { // Arrange var calculator = new EvidenceWeightedScoreCalculator(); var input = CreateTestInput("throughput-test"); // Warmup for (var i = 0; i < WarmupIterations; i++) { calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); } // Act - Run for 1 second and count operations var count = 0; var sw = Stopwatch.StartNew(); while (sw.ElapsedMilliseconds < 1000) { calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction); count++; } sw.Stop(); var opsPerSecond = count * 1000.0 / sw.ElapsedMilliseconds; // Assert - should exceed 1000 ops/sec (actual should be 10000+) opsPerSecond.Should().BeGreaterThan(1000, $"throughput should exceed 1000 evaluations/sec (actual: {opsPerSecond:F0} ops/sec)"); } #endregion #region Normalizer Pipeline Performance Tests [Fact(DisplayName = "Normalizer aggregation completes under 5ms")] public void NormalizerAggregation_CompletesUnder5ms() { // Arrange var aggregator = new NormalizerAggregator(); var evidence = CreateTestEvidence("norm-perf"); // Warmup for (var i = 0; i < WarmupIterations; i++) { aggregator.Aggregate(evidence); } // Act var sw = Stopwatch.StartNew(); var result = aggregator.Aggregate(evidence); sw.Stop(); // Assert result.Should().NotBeNull(); sw.ElapsedMilliseconds.Should().BeLessThan(5, $"normalizer aggregation should complete in under 5ms (actual: {sw.ElapsedMilliseconds}ms)"); } #endregion #region Test Helpers private static EvidenceWeightedScoreInput CreateTestInput(string findingId) { return new EvidenceWeightedScoreInput { FindingId = findingId, Rch = 0.75, Rts = 0.60, Bkp = 0.40, Xpl = 0.55, Src = 0.65, Mit = 0.20 }; } private static FindingEvidence CreateTestEvidence(string findingId) { return new FindingEvidence { FindingId = findingId, Reachability = new ReachabilityInput { State = global::StellaOps.Signals.EvidenceWeightedScore.ReachabilityState.DynamicReachable, Confidence = 0.85 }, Runtime = new RuntimeInput { Posture = global::StellaOps.Signals.EvidenceWeightedScore.RuntimePosture.ActiveTracing, ObservationCount = 3, RecencyFactor = 0.75 }, Exploit = new ExploitInput { EpssScore = 0.45, EpssPercentile = 75, KevStatus = KevStatus.NotInKev, PublicExploitAvailable = false } }; } #endregion }