master
96d52884e8
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implemented PolicyDslValidator with command-line options for strict mode and JSON output. - Created PolicySchemaExporter to generate JSON schemas for policy-related models. - Developed PolicySimulationSmoke tool to validate policy simulations against expected outcomes. - Added project files and necessary dependencies for each tool. - Ensured proper error handling and usage instructions across tools.
377 lines
17 KiB
C#
377 lines
17 KiB
C#
using System.Globalization;
|
|
using StellaOps.Bench.LinkNotMerge.Vex.Baseline;
|
|
using StellaOps.Bench.LinkNotMerge.Vex.Reporting;
|
|
|
|
namespace StellaOps.Bench.LinkNotMerge.Vex;
|
|
|
|
internal static class Program
|
|
{
|
|
public static async Task<int> Main(string[] args)
|
|
{
|
|
try
|
|
{
|
|
var options = ProgramOptions.Parse(args);
|
|
var config = await VexBenchmarkConfig.LoadAsync(options.ConfigPath).ConfigureAwait(false);
|
|
var baseline = await BaselineLoader.LoadAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
var results = new List<VexScenarioResult>();
|
|
var reports = new List<BenchmarkScenarioReport>();
|
|
var failures = new List<string>();
|
|
|
|
foreach (var scenario in config.Scenarios)
|
|
{
|
|
var iterations = scenario.ResolveIterations(config.Iterations);
|
|
var runner = new VexScenarioRunner(scenario);
|
|
var execution = runner.Execute(iterations, CancellationToken.None);
|
|
|
|
var totalStats = DurationStatistics.From(execution.TotalDurationsMs);
|
|
var insertStats = DurationStatistics.From(execution.InsertDurationsMs);
|
|
var correlationStats = DurationStatistics.From(execution.CorrelationDurationsMs);
|
|
var allocationStats = AllocationStatistics.From(execution.AllocatedMb);
|
|
var observationThroughputStats = ThroughputStatistics.From(execution.ObservationThroughputsPerSecond);
|
|
var eventThroughputStats = ThroughputStatistics.From(execution.EventThroughputsPerSecond);
|
|
|
|
var thresholdMs = scenario.ThresholdMs ?? options.ThresholdMs ?? config.ThresholdMs;
|
|
var observationFloor = scenario.MinThroughputPerSecond ?? options.MinThroughputPerSecond ?? config.MinThroughputPerSecond;
|
|
var eventFloor = scenario.MinEventThroughputPerSecond ?? options.MinEventThroughputPerSecond ?? config.MinEventThroughputPerSecond;
|
|
var allocationLimit = scenario.MaxAllocatedMb ?? options.MaxAllocatedMb ?? config.MaxAllocatedMb;
|
|
|
|
var result = new VexScenarioResult(
|
|
scenario.ScenarioId,
|
|
scenario.DisplayLabel,
|
|
iterations,
|
|
execution.ObservationCount,
|
|
execution.AliasGroups,
|
|
execution.StatementCount,
|
|
execution.EventCount,
|
|
totalStats,
|
|
insertStats,
|
|
correlationStats,
|
|
observationThroughputStats,
|
|
eventThroughputStats,
|
|
allocationStats,
|
|
thresholdMs,
|
|
observationFloor,
|
|
eventFloor,
|
|
allocationLimit);
|
|
|
|
results.Add(result);
|
|
|
|
if (thresholdMs is { } threshold && result.TotalStatistics.MaxMs > threshold)
|
|
{
|
|
failures.Add($"{result.Id} exceeded total latency threshold: {result.TotalStatistics.MaxMs:F2} ms > {threshold:F2} ms");
|
|
}
|
|
|
|
if (observationFloor is { } obsFloor && result.ObservationThroughputStatistics.MinPerSecond < obsFloor)
|
|
{
|
|
failures.Add($"{result.Id} fell below observation throughput floor: {result.ObservationThroughputStatistics.MinPerSecond:N0} obs/s < {obsFloor:N0} obs/s");
|
|
}
|
|
|
|
if (eventFloor is { } evtFloor && result.EventThroughputStatistics.MinPerSecond < evtFloor)
|
|
{
|
|
failures.Add($"{result.Id} fell below event throughput floor: {result.EventThroughputStatistics.MinPerSecond:N0} events/s < {evtFloor:N0} events/s");
|
|
}
|
|
|
|
if (allocationLimit is { } limit && result.AllocationStatistics.MaxAllocatedMb > limit)
|
|
{
|
|
failures.Add($"{result.Id} exceeded allocation budget: {result.AllocationStatistics.MaxAllocatedMb:F2} MB > {limit:F2} MB");
|
|
}
|
|
|
|
baseline.TryGetValue(result.Id, out var baselineEntry);
|
|
var report = new BenchmarkScenarioReport(result, baselineEntry, options.RegressionLimit);
|
|
reports.Add(report);
|
|
failures.AddRange(report.BuildRegressionFailureMessages());
|
|
}
|
|
|
|
TablePrinter.Print(results);
|
|
|
|
if (!string.IsNullOrWhiteSpace(options.CsvOutPath))
|
|
{
|
|
CsvWriter.Write(options.CsvOutPath!, results);
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(options.JsonOutPath))
|
|
{
|
|
var metadata = new BenchmarkJsonMetadata(
|
|
SchemaVersion: "linknotmerge-vex-bench/1.0",
|
|
CapturedAtUtc: (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime(),
|
|
Commit: options.Commit,
|
|
Environment: options.Environment);
|
|
|
|
await BenchmarkJsonWriter.WriteAsync(options.JsonOutPath!, metadata, reports, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath))
|
|
{
|
|
PrometheusWriter.Write(options.PrometheusOutPath!, reports);
|
|
}
|
|
|
|
if (failures.Count > 0)
|
|
{
|
|
Console.Error.WriteLine();
|
|
Console.Error.WriteLine("Benchmark failures detected:");
|
|
foreach (var failure in failures.Distinct())
|
|
{
|
|
Console.Error.WriteLine($" - {failure}");
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine($"linknotmerge-vex-bench error: {ex.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
private sealed record ProgramOptions(
|
|
string ConfigPath,
|
|
int? Iterations,
|
|
double? ThresholdMs,
|
|
double? MinThroughputPerSecond,
|
|
double? MinEventThroughputPerSecond,
|
|
double? MaxAllocatedMb,
|
|
string? CsvOutPath,
|
|
string? JsonOutPath,
|
|
string? PrometheusOutPath,
|
|
string BaselinePath,
|
|
DateTimeOffset? CapturedAtUtc,
|
|
string? Commit,
|
|
string? Environment,
|
|
double? RegressionLimit)
|
|
{
|
|
public static ProgramOptions Parse(string[] args)
|
|
{
|
|
var configPath = DefaultConfigPath();
|
|
var baselinePath = DefaultBaselinePath();
|
|
|
|
int? iterations = null;
|
|
double? thresholdMs = null;
|
|
double? minThroughput = null;
|
|
double? minEventThroughput = null;
|
|
double? maxAllocated = null;
|
|
string? csvOut = null;
|
|
string? jsonOut = null;
|
|
string? promOut = null;
|
|
DateTimeOffset? capturedAt = null;
|
|
string? commit = null;
|
|
string? environment = null;
|
|
double? regressionLimit = null;
|
|
|
|
for (var index = 0; index < args.Length; index++)
|
|
{
|
|
var current = args[index];
|
|
switch (current)
|
|
{
|
|
case "--config":
|
|
EnsureNext(args, index);
|
|
configPath = Path.GetFullPath(args[++index]);
|
|
break;
|
|
case "--iterations":
|
|
EnsureNext(args, index);
|
|
iterations = int.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
case "--threshold-ms":
|
|
EnsureNext(args, index);
|
|
thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
case "--min-throughput":
|
|
EnsureNext(args, index);
|
|
minThroughput = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
case "--min-event-throughput":
|
|
EnsureNext(args, index);
|
|
minEventThroughput = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
case "--max-allocated-mb":
|
|
EnsureNext(args, index);
|
|
maxAllocated = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
case "--csv":
|
|
EnsureNext(args, index);
|
|
csvOut = args[++index];
|
|
break;
|
|
case "--json":
|
|
EnsureNext(args, index);
|
|
jsonOut = args[++index];
|
|
break;
|
|
case "--prometheus":
|
|
EnsureNext(args, index);
|
|
promOut = args[++index];
|
|
break;
|
|
case "--baseline":
|
|
EnsureNext(args, index);
|
|
baselinePath = Path.GetFullPath(args[++index]);
|
|
break;
|
|
case "--captured-at":
|
|
EnsureNext(args, index);
|
|
capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal);
|
|
break;
|
|
case "--commit":
|
|
EnsureNext(args, index);
|
|
commit = args[++index];
|
|
break;
|
|
case "--environment":
|
|
EnsureNext(args, index);
|
|
environment = args[++index];
|
|
break;
|
|
case "--regression-limit":
|
|
EnsureNext(args, index);
|
|
regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
case "--help":
|
|
case "-h":
|
|
PrintUsage();
|
|
System.Environment.Exit(0);
|
|
break;
|
|
default:
|
|
throw new ArgumentException($"Unknown argument '{current}'.");
|
|
}
|
|
}
|
|
|
|
return new ProgramOptions(
|
|
configPath,
|
|
iterations,
|
|
thresholdMs,
|
|
minThroughput,
|
|
minEventThroughput,
|
|
maxAllocated,
|
|
csvOut,
|
|
jsonOut,
|
|
promOut,
|
|
baselinePath,
|
|
capturedAt,
|
|
commit,
|
|
environment,
|
|
regressionLimit);
|
|
}
|
|
|
|
private static string DefaultConfigPath()
|
|
{
|
|
var binaryDir = AppContext.BaseDirectory;
|
|
var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", ".."));
|
|
var benchRoot = Path.GetFullPath(Path.Combine(projectDir, ".."));
|
|
return Path.Combine(benchRoot, "config.json");
|
|
}
|
|
|
|
private static string DefaultBaselinePath()
|
|
{
|
|
var binaryDir = AppContext.BaseDirectory;
|
|
var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", ".."));
|
|
var benchRoot = Path.GetFullPath(Path.Combine(projectDir, ".."));
|
|
return Path.Combine(benchRoot, "baseline.csv");
|
|
}
|
|
|
|
private static void EnsureNext(string[] args, int index)
|
|
{
|
|
if (index + 1 >= args.Length)
|
|
{
|
|
throw new ArgumentException("Missing value for argument.");
|
|
}
|
|
}
|
|
|
|
private static void PrintUsage()
|
|
{
|
|
Console.WriteLine("Usage: linknotmerge-vex-bench [options]");
|
|
Console.WriteLine();
|
|
Console.WriteLine("Options:");
|
|
Console.WriteLine(" --config <path> Path to benchmark configuration JSON.");
|
|
Console.WriteLine(" --iterations <count> Override iteration count.");
|
|
Console.WriteLine(" --threshold-ms <value> Global latency threshold in milliseconds.");
|
|
Console.WriteLine(" --min-throughput <value> Observation throughput floor (observations/second).");
|
|
Console.WriteLine(" --min-event-throughput <value> Event emission throughput floor (events/second).");
|
|
Console.WriteLine(" --max-allocated-mb <value> Global allocation ceiling (MB).");
|
|
Console.WriteLine(" --csv <path> Write CSV results to path.");
|
|
Console.WriteLine(" --json <path> Write JSON results to path.");
|
|
Console.WriteLine(" --prometheus <path> Write Prometheus exposition metrics to path.");
|
|
Console.WriteLine(" --baseline <path> Baseline CSV path.");
|
|
Console.WriteLine(" --captured-at <iso8601> Timestamp to embed in JSON metadata.");
|
|
Console.WriteLine(" --commit <sha> Commit identifier for metadata.");
|
|
Console.WriteLine(" --environment <name> Environment label for metadata.");
|
|
Console.WriteLine(" --regression-limit <value> Regression multiplier (default 1.15).");
|
|
}
|
|
}
|
|
}
|
|
|
|
internal static class TablePrinter
|
|
{
|
|
public static void Print(IEnumerable<VexScenarioResult> results)
|
|
{
|
|
Console.WriteLine("Scenario | Observations | Statements | Events | Total(ms) | Correl(ms) | Insert(ms) | Obs k/s | Evnt k/s | Alloc(MB)");
|
|
Console.WriteLine("---------------------------- | ------------- | ---------- | ------- | ---------- | ---------- | ----------- | ------- | -------- | --------");
|
|
foreach (var row in results)
|
|
{
|
|
Console.WriteLine(string.Join(" | ", new[]
|
|
{
|
|
row.IdColumn,
|
|
row.ObservationsColumn,
|
|
row.StatementColumn,
|
|
row.EventColumn,
|
|
row.TotalMeanColumn,
|
|
row.CorrelationMeanColumn,
|
|
row.InsertMeanColumn,
|
|
row.ObservationThroughputColumn,
|
|
row.EventThroughputColumn,
|
|
row.AllocatedColumn,
|
|
}));
|
|
}
|
|
}
|
|
}
|
|
|
|
internal static class CsvWriter
|
|
{
|
|
public static void Write(string path, IEnumerable<VexScenarioResult> results)
|
|
{
|
|
ArgumentException.ThrowIfNullOrWhiteSpace(path);
|
|
ArgumentNullException.ThrowIfNull(results);
|
|
|
|
var resolved = Path.GetFullPath(path);
|
|
var directory = Path.GetDirectoryName(resolved);
|
|
if (!string.IsNullOrEmpty(directory))
|
|
{
|
|
Directory.CreateDirectory(directory);
|
|
}
|
|
|
|
using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None);
|
|
using var writer = new StreamWriter(stream);
|
|
writer.WriteLine("scenario,iterations,observations,statements,events,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_observation_throughput_per_sec,min_observation_throughput_per_sec,mean_event_throughput_per_sec,min_event_throughput_per_sec,max_allocated_mb");
|
|
|
|
foreach (var result in results)
|
|
{
|
|
writer.Write(result.Id);
|
|
writer.Write(',');
|
|
writer.Write(result.Iterations.ToString(CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.ObservationCount.ToString(CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.StatementCount.ToString(CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.EventCount.ToString(CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.TotalStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.TotalStatistics.P95Ms.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.TotalStatistics.MaxMs.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.InsertStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.CorrelationStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.ObservationThroughputStatistics.MeanPerSecond.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.ObservationThroughputStatistics.MinPerSecond.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.EventThroughputStatistics.MeanPerSecond.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.EventThroughputStatistics.MinPerSecond.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(result.AllocationStatistics.MaxAllocatedMb.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.WriteLine();
|
|
}
|
|
}
|
|
}
|