799f787de2
- Implemented PolicyDslValidator with command-line options for strict mode and JSON output. - Created PolicySchemaExporter to generate JSON schemas for policy-related models. - Developed PolicySimulationSmoke tool to validate policy simulations against expected outcomes. - Added project files and necessary dependencies for each tool. - Ensured proper error handling and usage instructions across tools.
84 lines
3.2 KiB
C#
84 lines
3.2 KiB
C#
using StellaOps.Bench.LinkNotMerge.Vex.Baseline;
|
|
using StellaOps.Bench.LinkNotMerge.Vex.Reporting;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Bench.LinkNotMerge.Vex.Tests;
|
|
|
|
public sealed class BenchmarkScenarioReportTests
|
|
{
|
|
[Fact]
|
|
public void RegressionDetection_FlagsBreaches()
|
|
{
|
|
var result = new VexScenarioResult(
|
|
Id: "scenario",
|
|
Label: "Scenario",
|
|
Iterations: 3,
|
|
ObservationCount: 1000,
|
|
AliasGroups: 100,
|
|
StatementCount: 6000,
|
|
EventCount: 3200,
|
|
TotalStatistics: new DurationStatistics(600, 700, 750),
|
|
InsertStatistics: new DurationStatistics(320, 360, 380),
|
|
CorrelationStatistics: new DurationStatistics(280, 320, 340),
|
|
ObservationThroughputStatistics: new ThroughputStatistics(8000, 7000),
|
|
EventThroughputStatistics: new ThroughputStatistics(3500, 3200),
|
|
AllocationStatistics: new AllocationStatistics(180),
|
|
ThresholdMs: null,
|
|
MinObservationThroughputPerSecond: null,
|
|
MinEventThroughputPerSecond: null,
|
|
MaxAllocatedThresholdMb: null);
|
|
|
|
var baseline = new BaselineEntry(
|
|
ScenarioId: "scenario",
|
|
Iterations: 3,
|
|
Observations: 1000,
|
|
Statements: 6000,
|
|
Events: 3200,
|
|
MeanTotalMs: 520,
|
|
P95TotalMs: 560,
|
|
MaxTotalMs: 580,
|
|
MeanInsertMs: 250,
|
|
MeanCorrelationMs: 260,
|
|
MeanObservationThroughputPerSecond: 9000,
|
|
MinObservationThroughputPerSecond: 8500,
|
|
MeanEventThroughputPerSecond: 4200,
|
|
MinEventThroughputPerSecond: 3800,
|
|
MaxAllocatedMb: 140);
|
|
|
|
var report = new BenchmarkScenarioReport(result, baseline, regressionLimit: 1.1);
|
|
|
|
Assert.True(report.DurationRegressionBreached);
|
|
Assert.True(report.ObservationThroughputRegressionBreached);
|
|
Assert.True(report.EventThroughputRegressionBreached);
|
|
Assert.Contains(report.BuildRegressionFailureMessages(), message => message.Contains("event throughput"));
|
|
}
|
|
|
|
[Fact]
|
|
public void RegressionDetection_NoBaseline_NoBreaches()
|
|
{
|
|
var result = new VexScenarioResult(
|
|
Id: "scenario",
|
|
Label: "Scenario",
|
|
Iterations: 3,
|
|
ObservationCount: 1000,
|
|
AliasGroups: 100,
|
|
StatementCount: 6000,
|
|
EventCount: 3200,
|
|
TotalStatistics: new DurationStatistics(480, 520, 540),
|
|
InsertStatistics: new DurationStatistics(260, 280, 300),
|
|
CorrelationStatistics: new DurationStatistics(220, 240, 260),
|
|
ObservationThroughputStatistics: new ThroughputStatistics(9000, 8800),
|
|
EventThroughputStatistics: new ThroughputStatistics(4200, 4100),
|
|
AllocationStatistics: new AllocationStatistics(150),
|
|
ThresholdMs: null,
|
|
MinObservationThroughputPerSecond: null,
|
|
MinEventThroughputPerSecond: null,
|
|
MaxAllocatedThresholdMb: null);
|
|
|
|
var report = new BenchmarkScenarioReport(result, baseline: null, regressionLimit: null);
|
|
|
|
Assert.False(report.RegressionBreached);
|
|
Assert.Empty(report.BuildRegressionFailureMessages());
|
|
}
|
|
}
|