337 lines
11 KiB
C#
337 lines
11 KiB
C#
// SPDX-License-Identifier: BUSL-1.1
|
|
// Copyright © 2025 StellaOps
|
|
// Sprint: SPRINT_8200_0012_0003_policy_engine_integration
|
|
// Task: PINT-8200-044 - Benchmark: policy evaluation < 50ms per finding
|
|
|
|
using System.Diagnostics;
|
|
using FluentAssertions;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using StellaOps.Policy.Engine.Scoring.EvidenceWeightedScore;
|
|
using StellaOps.Signals.EvidenceWeightedScore;
|
|
using StellaOps.Signals.EvidenceWeightedScore.Normalizers;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Policy.Engine.Tests.Benchmarks;
|
|
|
|
/// <summary>
|
|
/// Benchmark tests verifying that EWS calculation meets performance requirements.
|
|
/// Target: policy evaluation < 50ms per finding.
|
|
/// </summary>
|
|
[Trait("Category", "Benchmark")]
|
|
[Trait("Category", "Performance")]
|
|
[Trait("Sprint", "8200.0012.0003")]
|
|
[Trait("Task", "PINT-8200-044")]
|
|
public sealed class EwsCalculationBenchmarkTests
|
|
{
|
|
private const int TargetMaxMs = 50;
|
|
private const int WarmupIterations = 100;
|
|
private const int BenchmarkIterations = 1000;
|
|
|
|
private static ServiceCollection CreateServicesWithConfiguration()
|
|
{
|
|
var services = new ServiceCollection();
|
|
var configuration = new ConfigurationBuilder()
|
|
.AddInMemoryCollection()
|
|
.Build();
|
|
services.AddSingleton<IConfiguration>(configuration);
|
|
return services;
|
|
}
|
|
|
|
#region Calculator Performance Tests
|
|
|
|
[Fact(DisplayName = "Single EWS calculation completes under 50ms")]
|
|
public void SingleCalculation_CompletesUnder50ms()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var input = CreateTestInput("perf-single");
|
|
|
|
// Warmup
|
|
for (var i = 0; i < WarmupIterations; i++)
|
|
{
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
}
|
|
|
|
// Act
|
|
var sw = Stopwatch.StartNew();
|
|
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
sw.Stop();
|
|
|
|
// Assert
|
|
result.Should().NotBeNull();
|
|
sw.ElapsedMilliseconds.Should().BeLessThan(TargetMaxMs,
|
|
$"single EWS calculation should complete in under {TargetMaxMs}ms (actual: {sw.ElapsedMilliseconds}ms)");
|
|
}
|
|
|
|
[Fact(DisplayName = "Average calculation time over 1000 iterations is under 1ms")]
|
|
public void AverageCalculationTime_IsUnder1ms()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var input = CreateTestInput("perf-avg");
|
|
|
|
// Warmup
|
|
for (var i = 0; i < WarmupIterations; i++)
|
|
{
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
}
|
|
|
|
// Act
|
|
var sw = Stopwatch.StartNew();
|
|
for (var i = 0; i < BenchmarkIterations; i++)
|
|
{
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
}
|
|
sw.Stop();
|
|
|
|
var avgMs = (double)sw.ElapsedMilliseconds / BenchmarkIterations;
|
|
|
|
// Assert - average should be well under 1ms
|
|
avgMs.Should().BeLessThan(1.0,
|
|
$"average EWS calculation should be under 1ms (actual: {avgMs:F3}ms per calculation)");
|
|
}
|
|
|
|
[Fact(DisplayName = "P99 calculation time is under 10ms")]
|
|
public void P99CalculationTime_IsUnder10ms()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var input = CreateTestInput("perf-p99");
|
|
var timings = new long[BenchmarkIterations];
|
|
|
|
// Warmup
|
|
for (var i = 0; i < WarmupIterations; i++)
|
|
{
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
}
|
|
|
|
// Act - Collect timing for each iteration
|
|
var sw = new Stopwatch();
|
|
for (var i = 0; i < BenchmarkIterations; i++)
|
|
{
|
|
sw.Restart();
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
sw.Stop();
|
|
timings[i] = sw.ElapsedTicks;
|
|
}
|
|
|
|
// Calculate P99
|
|
Array.Sort(timings);
|
|
var p99Index = (int)(BenchmarkIterations * 0.99);
|
|
var p99Ticks = timings[p99Index];
|
|
var p99Ms = (double)p99Ticks / Stopwatch.Frequency * 1000;
|
|
|
|
// Assert
|
|
p99Ms.Should().BeLessThan(10.0,
|
|
$"P99 EWS calculation time should be under 10ms (actual: {p99Ms:F3}ms)");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Enricher Pipeline Performance Tests
|
|
|
|
[Fact(DisplayName = "Single enrichment completes under 50ms")]
|
|
public void SingleEnrichment_CompletesUnder50ms()
|
|
{
|
|
// Arrange
|
|
var services = CreateServicesWithConfiguration();
|
|
services.AddEvidenceWeightedScoring();
|
|
services.AddEvidenceNormalizers();
|
|
services.AddEvidenceWeightedScore(opts =>
|
|
{
|
|
opts.Enabled = true;
|
|
opts.EnableCaching = false; // Measure actual calculation
|
|
});
|
|
var provider = services.BuildServiceProvider();
|
|
var enricher = provider.GetRequiredService<IFindingScoreEnricher>();
|
|
var evidence = CreateTestEvidence("perf-enricher");
|
|
|
|
// Warmup
|
|
for (var i = 0; i < WarmupIterations; i++)
|
|
{
|
|
enricher.Enrich(evidence);
|
|
}
|
|
|
|
// Act
|
|
var sw = Stopwatch.StartNew();
|
|
var result = enricher.Enrich(evidence);
|
|
sw.Stop();
|
|
|
|
// Assert
|
|
result.Should().NotBeNull();
|
|
sw.ElapsedMilliseconds.Should().BeLessThan(TargetMaxMs,
|
|
$"single enrichment should complete in under {TargetMaxMs}ms (actual: {sw.ElapsedMilliseconds}ms)");
|
|
}
|
|
|
|
[Fact(DisplayName = "Enricher with caching improves performance")]
|
|
public void EnricherWithCaching_ImprovesPerformance()
|
|
{
|
|
// Arrange
|
|
var services = CreateServicesWithConfiguration();
|
|
services.AddEvidenceWeightedScoring();
|
|
services.AddEvidenceNormalizers();
|
|
services.AddEvidenceWeightedScore(opts =>
|
|
{
|
|
opts.Enabled = true;
|
|
opts.EnableCaching = true;
|
|
});
|
|
var provider = services.BuildServiceProvider();
|
|
var enricher = provider.GetRequiredService<IFindingScoreEnricher>();
|
|
var evidence = CreateTestEvidence("perf-cache");
|
|
|
|
// Warmup and populate cache
|
|
enricher.Enrich(evidence);
|
|
|
|
// Act - Measure cached access
|
|
var sw = Stopwatch.StartNew();
|
|
for (var i = 0; i < 100; i++)
|
|
{
|
|
enricher.Enrich(evidence);
|
|
}
|
|
sw.Stop();
|
|
|
|
var avgCachedMs = (double)sw.ElapsedMilliseconds / 100;
|
|
|
|
// Assert - cached access should be very fast
|
|
avgCachedMs.Should().BeLessThan(0.5,
|
|
$"cached enrichment should be under 0.5ms (actual: {avgCachedMs:F3}ms)");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Batch Performance Tests
|
|
|
|
[Fact(DisplayName = "Batch of 100 findings processes under 500ms")]
|
|
public void BatchOf100Findings_ProcessesUnder500ms()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var inputs = Enumerable.Range(0, 100)
|
|
.Select(i => CreateTestInput($"batch-{i}"))
|
|
.ToList();
|
|
|
|
// Warmup
|
|
foreach (var input in inputs.Take(10))
|
|
{
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
}
|
|
|
|
// Act
|
|
var sw = Stopwatch.StartNew();
|
|
foreach (var input in inputs)
|
|
{
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
}
|
|
sw.Stop();
|
|
|
|
// Assert - 100 findings * 50ms max = 5000ms, but should be much faster
|
|
sw.ElapsedMilliseconds.Should().BeLessThan(500,
|
|
$"batch of 100 findings should process in under 500ms (actual: {sw.ElapsedMilliseconds}ms)");
|
|
}
|
|
|
|
[Fact(DisplayName = "Throughput exceeds 1000 evaluations per second")]
|
|
public void Throughput_Exceeds1000PerSecond()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var input = CreateTestInput("throughput-test");
|
|
|
|
// Warmup
|
|
for (var i = 0; i < WarmupIterations; i++)
|
|
{
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
}
|
|
|
|
// Act - Run for 1 second and count operations
|
|
var count = 0;
|
|
var sw = Stopwatch.StartNew();
|
|
while (sw.ElapsedMilliseconds < 1000)
|
|
{
|
|
calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
count++;
|
|
}
|
|
sw.Stop();
|
|
|
|
var opsPerSecond = count * 1000.0 / sw.ElapsedMilliseconds;
|
|
|
|
// Assert - should exceed 1000 ops/sec (actual should be 10000+)
|
|
opsPerSecond.Should().BeGreaterThan(1000,
|
|
$"throughput should exceed 1000 evaluations/sec (actual: {opsPerSecond:F0} ops/sec)");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Normalizer Pipeline Performance Tests
|
|
|
|
[Fact(DisplayName = "Normalizer aggregation completes under 5ms")]
|
|
public void NormalizerAggregation_CompletesUnder5ms()
|
|
{
|
|
// Arrange
|
|
var aggregator = new NormalizerAggregator();
|
|
var evidence = CreateTestEvidence("norm-perf");
|
|
|
|
// Warmup
|
|
for (var i = 0; i < WarmupIterations; i++)
|
|
{
|
|
aggregator.Aggregate(evidence);
|
|
}
|
|
|
|
// Act
|
|
var sw = Stopwatch.StartNew();
|
|
var result = aggregator.Aggregate(evidence);
|
|
sw.Stop();
|
|
|
|
// Assert
|
|
result.Should().NotBeNull();
|
|
sw.ElapsedMilliseconds.Should().BeLessThan(5,
|
|
$"normalizer aggregation should complete in under 5ms (actual: {sw.ElapsedMilliseconds}ms)");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Test Helpers
|
|
|
|
private static EvidenceWeightedScoreInput CreateTestInput(string findingId)
|
|
{
|
|
return new EvidenceWeightedScoreInput
|
|
{
|
|
FindingId = findingId,
|
|
Rch = 0.75,
|
|
Rts = 0.60,
|
|
Bkp = 0.40,
|
|
Xpl = 0.55,
|
|
Src = 0.65,
|
|
Mit = 0.20
|
|
};
|
|
}
|
|
|
|
private static FindingEvidence CreateTestEvidence(string findingId)
|
|
{
|
|
return new FindingEvidence
|
|
{
|
|
FindingId = findingId,
|
|
Reachability = new ReachabilityInput
|
|
{
|
|
State = global::StellaOps.Signals.EvidenceWeightedScore.ReachabilityState.DynamicReachable,
|
|
Confidence = 0.85
|
|
},
|
|
Runtime = new RuntimeInput
|
|
{
|
|
Posture = global::StellaOps.Signals.EvidenceWeightedScore.RuntimePosture.ActiveTracing,
|
|
ObservationCount = 3,
|
|
RecencyFactor = 0.75
|
|
},
|
|
Exploit = new ExploitInput
|
|
{
|
|
EpssScore = 0.45,
|
|
EpssPercentile = 75,
|
|
KevStatus = KevStatus.NotInKev,
|
|
PublicExploitAvailable = false
|
|
}
|
|
};
|
|
}
|
|
|
|
#endregion
|
|
}
|