f4d7a15a00
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS. - Added unit tests for RustFsArtifactObjectStore functionality. - Created a RustFS migrator tool to transfer objects from S3 to RustFS. - Introduced policy preview and report models for API integration. - Added fixtures and tests for policy preview and report functionality. - Included necessary metadata and scripts for cache_pkg package.
394 lines
14 KiB
C#
394 lines
14 KiB
C#
using System.Globalization;
|
|
using StellaOps.Bench.ScannerAnalyzers.Baseline;
|
|
using StellaOps.Bench.ScannerAnalyzers.Reporting;
|
|
using StellaOps.Bench.ScannerAnalyzers.Scenarios;
|
|
|
|
namespace StellaOps.Bench.ScannerAnalyzers;
|
|
|
|
internal static class Program
|
|
{
|
|
public static async Task<int> Main(string[] args)
|
|
{
|
|
try
|
|
{
|
|
var options = ProgramOptions.Parse(args);
|
|
var config = await BenchmarkConfig.LoadAsync(options.ConfigPath).ConfigureAwait(false);
|
|
|
|
var iterations = options.Iterations ?? config.Iterations ?? 5;
|
|
var thresholdMs = options.ThresholdMs ?? config.ThresholdMs ?? 5000;
|
|
var repoRoot = ResolveRepoRoot(options.RepoRoot, options.ConfigPath);
|
|
var regressionLimit = options.RegressionLimit ?? 1.2d;
|
|
var capturedAt = (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime();
|
|
|
|
var baseline = await LoadBaselineDictionaryAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
var results = new List<ScenarioResult>();
|
|
var reports = new List<BenchmarkScenarioReport>();
|
|
var failures = new List<string>();
|
|
|
|
foreach (var scenario in config.Scenarios)
|
|
{
|
|
var runner = ScenarioRunnerFactory.Create(scenario);
|
|
var scenarioRoot = ResolveScenarioRoot(repoRoot, scenario.Root!);
|
|
|
|
var execution = await runner.ExecuteAsync(scenarioRoot, iterations, CancellationToken.None).ConfigureAwait(false);
|
|
var stats = ScenarioStatistics.FromDurations(execution.Durations);
|
|
var scenarioThreshold = scenario.ThresholdMs ?? thresholdMs;
|
|
|
|
var result = new ScenarioResult(
|
|
scenario.Id!,
|
|
scenario.Label ?? scenario.Id!,
|
|
execution.SampleCount,
|
|
stats.MeanMs,
|
|
stats.P95Ms,
|
|
stats.MaxMs,
|
|
iterations,
|
|
scenarioThreshold);
|
|
|
|
results.Add(result);
|
|
|
|
if (stats.MaxMs > scenarioThreshold)
|
|
{
|
|
failures.Add($"{scenario.Id} exceeded threshold: {stats.MaxMs:F2} ms > {scenarioThreshold:F2} ms");
|
|
}
|
|
|
|
baseline.TryGetValue(result.Id, out var baselineEntry);
|
|
var report = new BenchmarkScenarioReport(result, baselineEntry, regressionLimit);
|
|
if (report.BuildRegressionFailureMessage() is { } regressionFailure)
|
|
{
|
|
failures.Add(regressionFailure);
|
|
}
|
|
|
|
reports.Add(report);
|
|
}
|
|
|
|
TablePrinter.Print(results);
|
|
|
|
if (!string.IsNullOrWhiteSpace(options.CsvOutPath))
|
|
{
|
|
CsvWriter.Write(options.CsvOutPath!, results);
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(options.JsonOutPath))
|
|
{
|
|
var metadata = new BenchmarkJsonMetadata(
|
|
"1.0",
|
|
capturedAt,
|
|
options.Commit,
|
|
options.Environment);
|
|
|
|
await BenchmarkJsonWriter.WriteAsync(options.JsonOutPath!, metadata, reports, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath))
|
|
{
|
|
PrometheusWriter.Write(options.PrometheusOutPath!, reports);
|
|
}
|
|
|
|
if (failures.Count > 0)
|
|
{
|
|
Console.Error.WriteLine();
|
|
Console.Error.WriteLine("Performance threshold exceeded:");
|
|
foreach (var failure in failures)
|
|
{
|
|
Console.Error.WriteLine($" - {failure}");
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine(ex.Message);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
private static async Task<IReadOnlyDictionary<string, BaselineEntry>> LoadBaselineDictionaryAsync(string? baselinePath, CancellationToken cancellationToken)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(baselinePath))
|
|
{
|
|
return new Dictionary<string, BaselineEntry>(StringComparer.OrdinalIgnoreCase);
|
|
}
|
|
|
|
var resolved = Path.GetFullPath(baselinePath);
|
|
if (!File.Exists(resolved))
|
|
{
|
|
return new Dictionary<string, BaselineEntry>(StringComparer.OrdinalIgnoreCase);
|
|
}
|
|
|
|
return await BaselineLoader.LoadAsync(resolved, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private static string ResolveRepoRoot(string? overridePath, string configPath)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(overridePath))
|
|
{
|
|
return Path.GetFullPath(overridePath);
|
|
}
|
|
|
|
var configDirectory = Path.GetDirectoryName(configPath);
|
|
if (string.IsNullOrWhiteSpace(configDirectory))
|
|
{
|
|
return Directory.GetCurrentDirectory();
|
|
}
|
|
|
|
return Path.GetFullPath(Path.Combine(configDirectory, "..", ".."));
|
|
}
|
|
|
|
private static string ResolveScenarioRoot(string repoRoot, string relativeRoot)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(relativeRoot))
|
|
{
|
|
throw new InvalidOperationException("Scenario root is required.");
|
|
}
|
|
|
|
var combined = Path.GetFullPath(Path.Combine(repoRoot, relativeRoot));
|
|
if (!PathUtilities.IsWithinRoot(repoRoot, combined))
|
|
{
|
|
throw new InvalidOperationException($"Scenario root '{relativeRoot}' escapes repository root '{repoRoot}'.");
|
|
}
|
|
|
|
if (!Directory.Exists(combined))
|
|
{
|
|
throw new DirectoryNotFoundException($"Scenario root '{combined}' does not exist.");
|
|
}
|
|
|
|
return combined;
|
|
}
|
|
|
|
private sealed record ProgramOptions(
|
|
string ConfigPath,
|
|
int? Iterations,
|
|
double? ThresholdMs,
|
|
string? CsvOutPath,
|
|
string? JsonOutPath,
|
|
string? PrometheusOutPath,
|
|
string? RepoRoot,
|
|
string? BaselinePath,
|
|
DateTimeOffset? CapturedAtUtc,
|
|
string? Commit,
|
|
string? Environment,
|
|
double? RegressionLimit)
|
|
{
|
|
public static ProgramOptions Parse(string[] args)
|
|
{
|
|
var configPath = DefaultConfigPath();
|
|
var baselinePath = DefaultBaselinePath();
|
|
int? iterations = null;
|
|
double? thresholdMs = null;
|
|
string? csvOut = null;
|
|
string? jsonOut = null;
|
|
string? promOut = null;
|
|
string? repoRoot = null;
|
|
DateTimeOffset? capturedAt = null;
|
|
string? commit = null;
|
|
string? environment = null;
|
|
double? regressionLimit = null;
|
|
|
|
for (var index = 0; index < args.Length; index++)
|
|
{
|
|
var current = args[index];
|
|
switch (current)
|
|
{
|
|
case "--config":
|
|
EnsureNext(args, index);
|
|
configPath = Path.GetFullPath(args[++index]);
|
|
break;
|
|
case "--iterations":
|
|
EnsureNext(args, index);
|
|
iterations = int.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
case "--threshold-ms":
|
|
EnsureNext(args, index);
|
|
thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
case "--out":
|
|
case "--csv":
|
|
EnsureNext(args, index);
|
|
csvOut = args[++index];
|
|
break;
|
|
case "--json":
|
|
EnsureNext(args, index);
|
|
jsonOut = args[++index];
|
|
break;
|
|
case "--prom":
|
|
case "--prometheus":
|
|
EnsureNext(args, index);
|
|
promOut = args[++index];
|
|
break;
|
|
case "--baseline":
|
|
EnsureNext(args, index);
|
|
baselinePath = args[++index];
|
|
break;
|
|
case "--repo-root":
|
|
case "--samples":
|
|
EnsureNext(args, index);
|
|
repoRoot = args[++index];
|
|
break;
|
|
case "--captured-at":
|
|
EnsureNext(args, index);
|
|
capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
|
|
break;
|
|
case "--commit":
|
|
EnsureNext(args, index);
|
|
commit = args[++index];
|
|
break;
|
|
case "--environment":
|
|
EnsureNext(args, index);
|
|
environment = args[++index];
|
|
break;
|
|
case "--regression-limit":
|
|
EnsureNext(args, index);
|
|
regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
|
break;
|
|
default:
|
|
throw new ArgumentException($"Unknown argument: {current}", nameof(args));
|
|
}
|
|
}
|
|
|
|
return new ProgramOptions(configPath, iterations, thresholdMs, csvOut, jsonOut, promOut, repoRoot, baselinePath, capturedAt, commit, environment, regressionLimit);
|
|
}
|
|
|
|
private static string DefaultConfigPath()
|
|
{
|
|
var binaryDir = AppContext.BaseDirectory;
|
|
var projectRoot = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", ".."));
|
|
var configDirectory = Path.GetFullPath(Path.Combine(projectRoot, ".."));
|
|
return Path.Combine(configDirectory, "config.json");
|
|
}
|
|
|
|
private static string? DefaultBaselinePath()
|
|
{
|
|
var binaryDir = AppContext.BaseDirectory;
|
|
var projectRoot = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", ".."));
|
|
var benchRoot = Path.GetFullPath(Path.Combine(projectRoot, ".."));
|
|
var baselinePath = Path.Combine(benchRoot, "baseline.csv");
|
|
return File.Exists(baselinePath) ? baselinePath : baselinePath;
|
|
}
|
|
|
|
private static void EnsureNext(string[] args, int index)
|
|
{
|
|
if (index + 1 >= args.Length)
|
|
{
|
|
throw new ArgumentException("Missing value for argument.", nameof(args));
|
|
}
|
|
}
|
|
}
|
|
|
|
private sealed record ScenarioStatistics(double MeanMs, double P95Ms, double MaxMs)
|
|
{
|
|
public static ScenarioStatistics FromDurations(IReadOnlyList<double> durations)
|
|
{
|
|
if (durations.Count == 0)
|
|
{
|
|
return new ScenarioStatistics(0, 0, 0);
|
|
}
|
|
|
|
var sorted = durations.ToArray();
|
|
Array.Sort(sorted);
|
|
|
|
var total = 0d;
|
|
foreach (var value in durations)
|
|
{
|
|
total += value;
|
|
}
|
|
|
|
var mean = total / durations.Count;
|
|
var p95 = Percentile(sorted, 95);
|
|
var max = sorted[^1];
|
|
|
|
return new ScenarioStatistics(mean, p95, max);
|
|
}
|
|
|
|
private static double Percentile(IReadOnlyList<double> sorted, double percentile)
|
|
{
|
|
if (sorted.Count == 0)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
var rank = (percentile / 100d) * (sorted.Count - 1);
|
|
var lower = (int)Math.Floor(rank);
|
|
var upper = (int)Math.Ceiling(rank);
|
|
var weight = rank - lower;
|
|
|
|
if (upper >= sorted.Count)
|
|
{
|
|
return sorted[lower];
|
|
}
|
|
|
|
return sorted[lower] + weight * (sorted[upper] - sorted[lower]);
|
|
}
|
|
}
|
|
|
|
private static class TablePrinter
|
|
{
|
|
public static void Print(IEnumerable<ScenarioResult> results)
|
|
{
|
|
Console.WriteLine("Scenario | Count | Mean(ms) | P95(ms) | Max(ms)");
|
|
Console.WriteLine("---------------------------- | ----- | --------- | --------- | ----------");
|
|
foreach (var row in results)
|
|
{
|
|
Console.WriteLine(string.Join(" | ", new[]
|
|
{
|
|
row.IdColumn,
|
|
row.SampleCountColumn,
|
|
row.MeanColumn,
|
|
row.P95Column,
|
|
row.MaxColumn
|
|
}));
|
|
}
|
|
}
|
|
}
|
|
|
|
private static class CsvWriter
|
|
{
|
|
public static void Write(string path, IEnumerable<ScenarioResult> results)
|
|
{
|
|
var resolvedPath = Path.GetFullPath(path);
|
|
var directory = Path.GetDirectoryName(resolvedPath);
|
|
if (!string.IsNullOrEmpty(directory))
|
|
{
|
|
Directory.CreateDirectory(directory);
|
|
}
|
|
|
|
using var stream = new FileStream(resolvedPath, FileMode.Create, FileAccess.Write, FileShare.None);
|
|
using var writer = new StreamWriter(stream);
|
|
writer.WriteLine("scenario,iterations,sample_count,mean_ms,p95_ms,max_ms");
|
|
|
|
foreach (var row in results)
|
|
{
|
|
writer.Write(row.Id);
|
|
writer.Write(',');
|
|
writer.Write(row.Iterations.ToString(CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(row.SampleCount.ToString(CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(row.MeanMs.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(row.P95Ms.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.Write(',');
|
|
writer.Write(row.MaxMs.ToString("F4", CultureInfo.InvariantCulture));
|
|
writer.WriteLine();
|
|
}
|
|
}
|
|
}
|
|
|
|
internal static class PathUtilities
|
|
{
|
|
public static bool IsWithinRoot(string root, string candidate)
|
|
{
|
|
var relative = Path.GetRelativePath(root, candidate);
|
|
if (string.IsNullOrEmpty(relative) || relative == ".")
|
|
{
|
|
return true;
|
|
}
|
|
|
|
return !relative.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relative);
|
|
}
|
|
}
|
|
}
|