feat: Add RustFS artifact object store and migration tool
- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS. - Added unit tests for RustFsArtifactObjectStore functionality. - Created a RustFS migrator tool to transfer objects from S3 to RustFS. - Introduced policy preview and report models for API integration. - Added fixtures and tests for policy preview and report functionality. - Included necessary metadata and scripts for cache_pkg package.
This commit is contained in:
		@@ -0,0 +1,9 @@
 | 
			
		||||
namespace StellaOps.Bench.ScannerAnalyzers.Baseline;
 | 
			
		||||
 | 
			
		||||
internal sealed record BaselineEntry(
 | 
			
		||||
    string ScenarioId,
 | 
			
		||||
    int Iterations,
 | 
			
		||||
    int SampleCount,
 | 
			
		||||
    double MeanMs,
 | 
			
		||||
    double P95Ms,
 | 
			
		||||
    double MaxMs);
 | 
			
		||||
@@ -0,0 +1,88 @@
 | 
			
		||||
using System.Globalization;
 | 
			
		||||
 | 
			
		||||
namespace StellaOps.Bench.ScannerAnalyzers.Baseline;
 | 
			
		||||
 | 
			
		||||
internal static class BaselineLoader
 | 
			
		||||
{
 | 
			
		||||
    public static async Task<IReadOnlyDictionary<string, BaselineEntry>> LoadAsync(string path, CancellationToken cancellationToken)
 | 
			
		||||
    {
 | 
			
		||||
        if (string.IsNullOrWhiteSpace(path))
 | 
			
		||||
        {
 | 
			
		||||
            throw new ArgumentException("Baseline path must be provided.", nameof(path));
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        var resolved = Path.GetFullPath(path);
 | 
			
		||||
        if (!File.Exists(resolved))
 | 
			
		||||
        {
 | 
			
		||||
            throw new FileNotFoundException($"Baseline file not found at {resolved}", resolved);
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        var result = new Dictionary<string, BaselineEntry>(StringComparer.OrdinalIgnoreCase);
 | 
			
		||||
 | 
			
		||||
        await using var stream = new FileStream(resolved, FileMode.Open, FileAccess.Read, FileShare.Read);
 | 
			
		||||
        using var reader = new StreamReader(stream);
 | 
			
		||||
        string? line;
 | 
			
		||||
        var isFirst = true;
 | 
			
		||||
 | 
			
		||||
        while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null)
 | 
			
		||||
        {
 | 
			
		||||
            cancellationToken.ThrowIfCancellationRequested();
 | 
			
		||||
            if (string.IsNullOrWhiteSpace(line))
 | 
			
		||||
            {
 | 
			
		||||
                continue;
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            if (isFirst)
 | 
			
		||||
            {
 | 
			
		||||
                isFirst = false;
 | 
			
		||||
                if (line.StartsWith("scenario,", StringComparison.OrdinalIgnoreCase))
 | 
			
		||||
                {
 | 
			
		||||
                    continue;
 | 
			
		||||
                }
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            var entry = ParseLine(line);
 | 
			
		||||
            result[entry.ScenarioId] = entry;
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        return result;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static BaselineEntry ParseLine(string line)
 | 
			
		||||
    {
 | 
			
		||||
        var parts = line.Split(',', StringSplitOptions.TrimEntries);
 | 
			
		||||
        if (parts.Length < 6)
 | 
			
		||||
        {
 | 
			
		||||
            throw new InvalidDataException($"Baseline CSV row malformed: '{line}'");
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        var scenarioId = parts[0];
 | 
			
		||||
        var iterations = ParseInt(parts[1], nameof(BaselineEntry.Iterations));
 | 
			
		||||
        var sampleCount = ParseInt(parts[2], nameof(BaselineEntry.SampleCount));
 | 
			
		||||
        var meanMs = ParseDouble(parts[3], nameof(BaselineEntry.MeanMs));
 | 
			
		||||
        var p95Ms = ParseDouble(parts[4], nameof(BaselineEntry.P95Ms));
 | 
			
		||||
        var maxMs = ParseDouble(parts[5], nameof(BaselineEntry.MaxMs));
 | 
			
		||||
 | 
			
		||||
        return new BaselineEntry(scenarioId, iterations, sampleCount, meanMs, p95Ms, maxMs);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static int ParseInt(string value, string field)
 | 
			
		||||
    {
 | 
			
		||||
        if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
 | 
			
		||||
        {
 | 
			
		||||
            throw new InvalidDataException($"Failed to parse integer {field} from '{value}'.");
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        return parsed;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static double ParseDouble(string value, string field)
 | 
			
		||||
    {
 | 
			
		||||
        if (!double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed))
 | 
			
		||||
        {
 | 
			
		||||
            throw new InvalidDataException($"Failed to parse double {field} from '{value}'.");
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        return parsed;
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@@ -1,4 +1,6 @@
 | 
			
		||||
using System.Globalization;
 | 
			
		||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
 | 
			
		||||
using StellaOps.Bench.ScannerAnalyzers.Reporting;
 | 
			
		||||
using StellaOps.Bench.ScannerAnalyzers.Scenarios;
 | 
			
		||||
 | 
			
		||||
namespace StellaOps.Bench.ScannerAnalyzers;
 | 
			
		||||
@@ -15,8 +17,13 @@ internal static class Program
 | 
			
		||||
            var iterations = options.Iterations ?? config.Iterations ?? 5;
 | 
			
		||||
            var thresholdMs = options.ThresholdMs ?? config.ThresholdMs ?? 5000;
 | 
			
		||||
            var repoRoot = ResolveRepoRoot(options.RepoRoot, options.ConfigPath);
 | 
			
		||||
            var regressionLimit = options.RegressionLimit ?? 1.2d;
 | 
			
		||||
            var capturedAt = (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime();
 | 
			
		||||
 | 
			
		||||
            var baseline = await LoadBaselineDictionaryAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false);
 | 
			
		||||
 | 
			
		||||
            var results = new List<ScenarioResult>();
 | 
			
		||||
            var reports = new List<BenchmarkScenarioReport>();
 | 
			
		||||
            var failures = new List<string>();
 | 
			
		||||
 | 
			
		||||
            foreach (var scenario in config.Scenarios)
 | 
			
		||||
@@ -28,26 +35,54 @@ internal static class Program
 | 
			
		||||
                var stats = ScenarioStatistics.FromDurations(execution.Durations);
 | 
			
		||||
                var scenarioThreshold = scenario.ThresholdMs ?? thresholdMs;
 | 
			
		||||
 | 
			
		||||
                results.Add(new ScenarioResult(
 | 
			
		||||
                var result = new ScenarioResult(
 | 
			
		||||
                    scenario.Id!,
 | 
			
		||||
                    scenario.Label ?? scenario.Id!,
 | 
			
		||||
                    execution.SampleCount,
 | 
			
		||||
                    stats.MeanMs,
 | 
			
		||||
                    stats.P95Ms,
 | 
			
		||||
                    stats.MaxMs,
 | 
			
		||||
                    iterations));
 | 
			
		||||
                    iterations,
 | 
			
		||||
                    scenarioThreshold);
 | 
			
		||||
 | 
			
		||||
                results.Add(result);
 | 
			
		||||
 | 
			
		||||
                if (stats.MaxMs > scenarioThreshold)
 | 
			
		||||
                {
 | 
			
		||||
                    failures.Add($"{scenario.Id} exceeded threshold: {stats.MaxMs:F2} ms > {scenarioThreshold:F2} ms");
 | 
			
		||||
                }
 | 
			
		||||
 | 
			
		||||
                baseline.TryGetValue(result.Id, out var baselineEntry);
 | 
			
		||||
                var report = new BenchmarkScenarioReport(result, baselineEntry, regressionLimit);
 | 
			
		||||
                if (report.BuildRegressionFailureMessage() is { } regressionFailure)
 | 
			
		||||
                {
 | 
			
		||||
                    failures.Add(regressionFailure);
 | 
			
		||||
                }
 | 
			
		||||
 | 
			
		||||
                reports.Add(report);
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            TablePrinter.Print(results);
 | 
			
		||||
 | 
			
		||||
            if (!string.IsNullOrWhiteSpace(options.OutPath))
 | 
			
		||||
            if (!string.IsNullOrWhiteSpace(options.CsvOutPath))
 | 
			
		||||
            {
 | 
			
		||||
                CsvWriter.Write(options.OutPath!, results);
 | 
			
		||||
                CsvWriter.Write(options.CsvOutPath!, results);
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            if (!string.IsNullOrWhiteSpace(options.JsonOutPath))
 | 
			
		||||
            {
 | 
			
		||||
                var metadata = new BenchmarkJsonMetadata(
 | 
			
		||||
                    "1.0",
 | 
			
		||||
                    capturedAt,
 | 
			
		||||
                    options.Commit,
 | 
			
		||||
                    options.Environment);
 | 
			
		||||
 | 
			
		||||
                await BenchmarkJsonWriter.WriteAsync(options.JsonOutPath!, metadata, reports, CancellationToken.None).ConfigureAwait(false);
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath))
 | 
			
		||||
            {
 | 
			
		||||
                PrometheusWriter.Write(options.PrometheusOutPath!, reports);
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            if (failures.Count > 0)
 | 
			
		||||
@@ -71,6 +106,22 @@ internal static class Program
 | 
			
		||||
        }
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static async Task<IReadOnlyDictionary<string, BaselineEntry>> LoadBaselineDictionaryAsync(string? baselinePath, CancellationToken cancellationToken)
 | 
			
		||||
    {
 | 
			
		||||
        if (string.IsNullOrWhiteSpace(baselinePath))
 | 
			
		||||
        {
 | 
			
		||||
            return new Dictionary<string, BaselineEntry>(StringComparer.OrdinalIgnoreCase);
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        var resolved = Path.GetFullPath(baselinePath);
 | 
			
		||||
        if (!File.Exists(resolved))
 | 
			
		||||
        {
 | 
			
		||||
            return new Dictionary<string, BaselineEntry>(StringComparer.OrdinalIgnoreCase);
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        return await BaselineLoader.LoadAsync(resolved, cancellationToken).ConfigureAwait(false);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static string ResolveRepoRoot(string? overridePath, string configPath)
 | 
			
		||||
    {
 | 
			
		||||
        if (!string.IsNullOrWhiteSpace(overridePath))
 | 
			
		||||
@@ -108,15 +159,34 @@ internal static class Program
 | 
			
		||||
        return combined;
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private sealed record ProgramOptions(string ConfigPath, int? Iterations, double? ThresholdMs, string? OutPath, string? RepoRoot)
 | 
			
		||||
    private sealed record ProgramOptions(
 | 
			
		||||
        string ConfigPath,
 | 
			
		||||
        int? Iterations,
 | 
			
		||||
        double? ThresholdMs,
 | 
			
		||||
        string? CsvOutPath,
 | 
			
		||||
        string? JsonOutPath,
 | 
			
		||||
        string? PrometheusOutPath,
 | 
			
		||||
        string? RepoRoot,
 | 
			
		||||
        string? BaselinePath,
 | 
			
		||||
        DateTimeOffset? CapturedAtUtc,
 | 
			
		||||
        string? Commit,
 | 
			
		||||
        string? Environment,
 | 
			
		||||
        double? RegressionLimit)
 | 
			
		||||
    {
 | 
			
		||||
        public static ProgramOptions Parse(string[] args)
 | 
			
		||||
        {
 | 
			
		||||
            var configPath = DefaultConfigPath();
 | 
			
		||||
            var baselinePath = DefaultBaselinePath();
 | 
			
		||||
            int? iterations = null;
 | 
			
		||||
            double? thresholdMs = null;
 | 
			
		||||
            string? outPath = null;
 | 
			
		||||
            string? csvOut = null;
 | 
			
		||||
            string? jsonOut = null;
 | 
			
		||||
            string? promOut = null;
 | 
			
		||||
            string? repoRoot = null;
 | 
			
		||||
            DateTimeOffset? capturedAt = null;
 | 
			
		||||
            string? commit = null;
 | 
			
		||||
            string? environment = null;
 | 
			
		||||
            double? regressionLimit = null;
 | 
			
		||||
 | 
			
		||||
            for (var index = 0; index < args.Length; index++)
 | 
			
		||||
            {
 | 
			
		||||
@@ -136,20 +206,50 @@ internal static class Program
 | 
			
		||||
                        thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture);
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--out":
 | 
			
		||||
                    case "--csv":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        outPath = args[++index];
 | 
			
		||||
                        csvOut = args[++index];
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--json":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        jsonOut = args[++index];
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--prom":
 | 
			
		||||
                    case "--prometheus":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        promOut = args[++index];
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--baseline":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        baselinePath = args[++index];
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--repo-root":
 | 
			
		||||
                    case "--samples":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        repoRoot = args[++index];
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--captured-at":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--commit":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        commit = args[++index];
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--environment":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        environment = args[++index];
 | 
			
		||||
                        break;
 | 
			
		||||
                    case "--regression-limit":
 | 
			
		||||
                        EnsureNext(args, index);
 | 
			
		||||
                        regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture);
 | 
			
		||||
                        break;
 | 
			
		||||
                    default:
 | 
			
		||||
                        throw new ArgumentException($"Unknown argument: {current}", nameof(args));
 | 
			
		||||
                }
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            return new ProgramOptions(configPath, iterations, thresholdMs, outPath, repoRoot);
 | 
			
		||||
            return new ProgramOptions(configPath, iterations, thresholdMs, csvOut, jsonOut, promOut, repoRoot, baselinePath, capturedAt, commit, environment, regressionLimit);
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        private static string DefaultConfigPath()
 | 
			
		||||
@@ -160,6 +260,15 @@ internal static class Program
 | 
			
		||||
            return Path.Combine(configDirectory, "config.json");
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        private static string? DefaultBaselinePath()
 | 
			
		||||
        {
 | 
			
		||||
            var binaryDir = AppContext.BaseDirectory;
 | 
			
		||||
            var projectRoot = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", ".."));
 | 
			
		||||
            var benchRoot = Path.GetFullPath(Path.Combine(projectRoot, ".."));
 | 
			
		||||
            var baselinePath = Path.Combine(benchRoot, "baseline.csv");
 | 
			
		||||
            return File.Exists(baselinePath) ? baselinePath : baselinePath;
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        private static void EnsureNext(string[] args, int index)
 | 
			
		||||
        {
 | 
			
		||||
            if (index + 1 >= args.Length)
 | 
			
		||||
@@ -169,15 +278,6 @@ internal static class Program
 | 
			
		||||
        }
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private sealed record ScenarioResult(
 | 
			
		||||
        string Id,
 | 
			
		||||
        string Label,
 | 
			
		||||
        int SampleCount,
 | 
			
		||||
        double MeanMs,
 | 
			
		||||
        double P95Ms,
 | 
			
		||||
        double MaxMs,
 | 
			
		||||
        int Iterations);
 | 
			
		||||
 | 
			
		||||
    private sealed record ScenarioStatistics(double MeanMs, double P95Ms, double MaxMs)
 | 
			
		||||
    {
 | 
			
		||||
        public static ScenarioStatistics FromDurations(IReadOnlyList<double> durations)
 | 
			
		||||
@@ -232,25 +332,16 @@ internal static class Program
 | 
			
		||||
            Console.WriteLine("---------------------------- | ----- | --------- | --------- | ----------");
 | 
			
		||||
            foreach (var row in results)
 | 
			
		||||
            {
 | 
			
		||||
                Console.WriteLine(FormatRow(row));
 | 
			
		||||
                Console.WriteLine(string.Join(" | ", new[]
 | 
			
		||||
                {
 | 
			
		||||
                    row.IdColumn,
 | 
			
		||||
                    row.SampleCountColumn,
 | 
			
		||||
                    row.MeanColumn,
 | 
			
		||||
                    row.P95Column,
 | 
			
		||||
                    row.MaxColumn
 | 
			
		||||
                }));
 | 
			
		||||
            }
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        private static string FormatRow(ScenarioResult row)
 | 
			
		||||
        {
 | 
			
		||||
            var idColumn = row.Id.Length <= 28
 | 
			
		||||
                ? row.Id.PadRight(28)
 | 
			
		||||
                : row.Id[..28];
 | 
			
		||||
 | 
			
		||||
            return string.Join(" | ", new[]
 | 
			
		||||
            {
 | 
			
		||||
                idColumn,
 | 
			
		||||
                row.SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5),
 | 
			
		||||
                row.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9),
 | 
			
		||||
                row.P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9),
 | 
			
		||||
                row.MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10),
 | 
			
		||||
            });
 | 
			
		||||
        }
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static class CsvWriter
 | 
			
		||||
 
 | 
			
		||||
@@ -0,0 +1,108 @@
 | 
			
		||||
using System.Text.Json;
 | 
			
		||||
using System.Text.Json.Serialization;
 | 
			
		||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
 | 
			
		||||
 | 
			
		||||
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
 | 
			
		||||
 | 
			
		||||
internal static class BenchmarkJsonWriter
 | 
			
		||||
{
 | 
			
		||||
    private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
 | 
			
		||||
    {
 | 
			
		||||
        WriteIndented = true,
 | 
			
		||||
        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
 | 
			
		||||
    };
 | 
			
		||||
 | 
			
		||||
    public static async Task WriteAsync(
 | 
			
		||||
        string path,
 | 
			
		||||
        BenchmarkJsonMetadata metadata,
 | 
			
		||||
        IReadOnlyList<BenchmarkScenarioReport> reports,
 | 
			
		||||
        CancellationToken cancellationToken)
 | 
			
		||||
    {
 | 
			
		||||
        ArgumentException.ThrowIfNullOrWhiteSpace(path);
 | 
			
		||||
        ArgumentNullException.ThrowIfNull(metadata);
 | 
			
		||||
        ArgumentNullException.ThrowIfNull(reports);
 | 
			
		||||
 | 
			
		||||
        var resolved = Path.GetFullPath(path);
 | 
			
		||||
        var directory = Path.GetDirectoryName(resolved);
 | 
			
		||||
        if (!string.IsNullOrEmpty(directory))
 | 
			
		||||
        {
 | 
			
		||||
            Directory.CreateDirectory(directory);
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        var document = new BenchmarkJsonDocument(
 | 
			
		||||
            metadata.SchemaVersion,
 | 
			
		||||
            metadata.CapturedAtUtc,
 | 
			
		||||
            metadata.Commit,
 | 
			
		||||
            metadata.Environment,
 | 
			
		||||
            reports.Select(CreateScenario).ToArray());
 | 
			
		||||
 | 
			
		||||
        await using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None);
 | 
			
		||||
        await JsonSerializer.SerializeAsync(stream, document, SerializerOptions, cancellationToken).ConfigureAwait(false);
 | 
			
		||||
        await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static BenchmarkJsonScenario CreateScenario(BenchmarkScenarioReport report)
 | 
			
		||||
    {
 | 
			
		||||
        var baseline = report.Baseline;
 | 
			
		||||
        return new BenchmarkJsonScenario(
 | 
			
		||||
            report.Result.Id,
 | 
			
		||||
            report.Result.Label,
 | 
			
		||||
            report.Result.Iterations,
 | 
			
		||||
            report.Result.SampleCount,
 | 
			
		||||
            report.Result.MeanMs,
 | 
			
		||||
            report.Result.P95Ms,
 | 
			
		||||
            report.Result.MaxMs,
 | 
			
		||||
            report.Result.ThresholdMs,
 | 
			
		||||
            baseline is null
 | 
			
		||||
                ? null
 | 
			
		||||
                : new BenchmarkJsonScenarioBaseline(
 | 
			
		||||
                    baseline.Iterations,
 | 
			
		||||
                    baseline.SampleCount,
 | 
			
		||||
                    baseline.MeanMs,
 | 
			
		||||
                    baseline.P95Ms,
 | 
			
		||||
                    baseline.MaxMs),
 | 
			
		||||
            new BenchmarkJsonScenarioRegression(
 | 
			
		||||
                report.MaxRegressionRatio,
 | 
			
		||||
                report.MeanRegressionRatio,
 | 
			
		||||
                report.RegressionLimit,
 | 
			
		||||
                report.RegressionBreached));
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private sealed record BenchmarkJsonDocument(
 | 
			
		||||
        string SchemaVersion,
 | 
			
		||||
        DateTimeOffset CapturedAt,
 | 
			
		||||
        string? Commit,
 | 
			
		||||
        string? Environment,
 | 
			
		||||
        IReadOnlyList<BenchmarkJsonScenario> Scenarios);
 | 
			
		||||
 | 
			
		||||
    private sealed record BenchmarkJsonScenario(
 | 
			
		||||
        string Id,
 | 
			
		||||
        string Label,
 | 
			
		||||
        int Iterations,
 | 
			
		||||
        int SampleCount,
 | 
			
		||||
        double MeanMs,
 | 
			
		||||
        double P95Ms,
 | 
			
		||||
        double MaxMs,
 | 
			
		||||
        double ThresholdMs,
 | 
			
		||||
        BenchmarkJsonScenarioBaseline? Baseline,
 | 
			
		||||
        BenchmarkJsonScenarioRegression Regression);
 | 
			
		||||
 | 
			
		||||
    private sealed record BenchmarkJsonScenarioBaseline(
 | 
			
		||||
        int Iterations,
 | 
			
		||||
        int SampleCount,
 | 
			
		||||
        double MeanMs,
 | 
			
		||||
        double P95Ms,
 | 
			
		||||
        double MaxMs);
 | 
			
		||||
 | 
			
		||||
    private sealed record BenchmarkJsonScenarioRegression(
 | 
			
		||||
        double? MaxRatio,
 | 
			
		||||
        double? MeanRatio,
 | 
			
		||||
        double Limit,
 | 
			
		||||
        bool Breached);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
internal sealed record BenchmarkJsonMetadata(
 | 
			
		||||
    string SchemaVersion,
 | 
			
		||||
    DateTimeOffset CapturedAtUtc,
 | 
			
		||||
    string? Commit,
 | 
			
		||||
    string? Environment);
 | 
			
		||||
@@ -0,0 +1,55 @@
 | 
			
		||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
 | 
			
		||||
 | 
			
		||||
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
 | 
			
		||||
 | 
			
		||||
internal sealed class BenchmarkScenarioReport
 | 
			
		||||
{
 | 
			
		||||
    private const double RegressionLimitDefault = 1.2d;
 | 
			
		||||
 | 
			
		||||
    public BenchmarkScenarioReport(ScenarioResult result, BaselineEntry? baseline, double? regressionLimit = null)
 | 
			
		||||
    {
 | 
			
		||||
        Result = result ?? throw new ArgumentNullException(nameof(result));
 | 
			
		||||
        Baseline = baseline;
 | 
			
		||||
        RegressionLimit = regressionLimit is { } limit && limit > 0 ? limit : RegressionLimitDefault;
 | 
			
		||||
        MaxRegressionRatio = CalculateRatio(result.MaxMs, baseline?.MaxMs);
 | 
			
		||||
        MeanRegressionRatio = CalculateRatio(result.MeanMs, baseline?.MeanMs);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    public ScenarioResult Result { get; }
 | 
			
		||||
 | 
			
		||||
    public BaselineEntry? Baseline { get; }
 | 
			
		||||
 | 
			
		||||
    public double RegressionLimit { get; }
 | 
			
		||||
 | 
			
		||||
    public double? MaxRegressionRatio { get; }
 | 
			
		||||
 | 
			
		||||
    public double? MeanRegressionRatio { get; }
 | 
			
		||||
 | 
			
		||||
    public bool RegressionBreached => MaxRegressionRatio.HasValue && MaxRegressionRatio.Value >= RegressionLimit;
 | 
			
		||||
 | 
			
		||||
    public string? BuildRegressionFailureMessage()
 | 
			
		||||
    {
 | 
			
		||||
        if (!RegressionBreached || MaxRegressionRatio is null)
 | 
			
		||||
        {
 | 
			
		||||
            return null;
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        var percentage = (MaxRegressionRatio.Value - 1d) * 100d;
 | 
			
		||||
        return $"{Result.Id} exceeded regression budget: max {Result.MaxMs:F2} ms vs baseline {Baseline!.MaxMs:F2} ms (+{percentage:F1}%)";
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static double? CalculateRatio(double current, double? baseline)
 | 
			
		||||
    {
 | 
			
		||||
        if (!baseline.HasValue)
 | 
			
		||||
        {
 | 
			
		||||
            return null;
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        if (baseline.Value <= 0d)
 | 
			
		||||
        {
 | 
			
		||||
            return null;
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        return current / baseline.Value;
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
@@ -0,0 +1,59 @@
 | 
			
		||||
using System.Globalization;
 | 
			
		||||
using System.Text;
 | 
			
		||||
 | 
			
		||||
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
 | 
			
		||||
 | 
			
		||||
internal static class PrometheusWriter
 | 
			
		||||
{
 | 
			
		||||
    public static void Write(string path, IReadOnlyList<BenchmarkScenarioReport> reports)
 | 
			
		||||
    {
 | 
			
		||||
        ArgumentException.ThrowIfNullOrWhiteSpace(path);
 | 
			
		||||
        ArgumentNullException.ThrowIfNull(reports);
 | 
			
		||||
 | 
			
		||||
        var resolved = Path.GetFullPath(path);
 | 
			
		||||
        var directory = Path.GetDirectoryName(resolved);
 | 
			
		||||
        if (!string.IsNullOrEmpty(directory))
 | 
			
		||||
        {
 | 
			
		||||
            Directory.CreateDirectory(directory);
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        var builder = new StringBuilder();
 | 
			
		||||
        builder.AppendLine("# HELP scanner_analyzer_bench_duration_ms Analyzer benchmark duration metrics in milliseconds.");
 | 
			
		||||
        builder.AppendLine("# TYPE scanner_analyzer_bench_duration_ms gauge");
 | 
			
		||||
 | 
			
		||||
        foreach (var report in reports)
 | 
			
		||||
        {
 | 
			
		||||
            var scenarioLabel = Escape(report.Result.Id);
 | 
			
		||||
            AppendMetric(builder, "scanner_analyzer_bench_mean_ms", scenarioLabel, report.Result.MeanMs);
 | 
			
		||||
            AppendMetric(builder, "scanner_analyzer_bench_p95_ms", scenarioLabel, report.Result.P95Ms);
 | 
			
		||||
            AppendMetric(builder, "scanner_analyzer_bench_max_ms", scenarioLabel, report.Result.MaxMs);
 | 
			
		||||
            AppendMetric(builder, "scanner_analyzer_bench_threshold_ms", scenarioLabel, report.Result.ThresholdMs);
 | 
			
		||||
 | 
			
		||||
            if (report.Baseline is { } baseline)
 | 
			
		||||
            {
 | 
			
		||||
                AppendMetric(builder, "scanner_analyzer_bench_baseline_max_ms", scenarioLabel, baseline.MaxMs);
 | 
			
		||||
                AppendMetric(builder, "scanner_analyzer_bench_baseline_mean_ms", scenarioLabel, baseline.MeanMs);
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            if (report.MaxRegressionRatio is { } ratio)
 | 
			
		||||
            {
 | 
			
		||||
                AppendMetric(builder, "scanner_analyzer_bench_regression_ratio", scenarioLabel, ratio);
 | 
			
		||||
                AppendMetric(builder, "scanner_analyzer_bench_regression_limit", scenarioLabel, report.RegressionLimit);
 | 
			
		||||
                AppendMetric(builder, "scanner_analyzer_bench_regression_breached", scenarioLabel, report.RegressionBreached ? 1 : 0);
 | 
			
		||||
            }
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static void AppendMetric(StringBuilder builder, string metric, string scenarioLabel, double value)
 | 
			
		||||
    {
 | 
			
		||||
        builder.Append(metric);
 | 
			
		||||
        builder.Append("{scenario=\"");
 | 
			
		||||
        builder.Append(scenarioLabel);
 | 
			
		||||
        builder.Append("\"} ");
 | 
			
		||||
        builder.AppendLine(value.ToString("G17", CultureInfo.InvariantCulture));
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    private static string Escape(string value) => value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal);
 | 
			
		||||
}
 | 
			
		||||
@@ -0,0 +1,24 @@
 | 
			
		||||
using System.Globalization;
 | 
			
		||||
 | 
			
		||||
namespace StellaOps.Bench.ScannerAnalyzers;
 | 
			
		||||
 | 
			
		||||
internal sealed record ScenarioResult(
 | 
			
		||||
    string Id,
 | 
			
		||||
    string Label,
 | 
			
		||||
    int SampleCount,
 | 
			
		||||
    double MeanMs,
 | 
			
		||||
    double P95Ms,
 | 
			
		||||
    double MaxMs,
 | 
			
		||||
    int Iterations,
 | 
			
		||||
    double ThresholdMs)
 | 
			
		||||
{
 | 
			
		||||
    public string IdColumn => Id.Length <= 28 ? Id.PadRight(28) : Id[..28];
 | 
			
		||||
 | 
			
		||||
    public string SampleCountColumn => SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5);
 | 
			
		||||
 | 
			
		||||
    public string MeanColumn => MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9);
 | 
			
		||||
 | 
			
		||||
    public string P95Column => P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9);
 | 
			
		||||
 | 
			
		||||
    public string MaxColumn => MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10);
 | 
			
		||||
}
 | 
			
		||||
@@ -8,6 +8,7 @@ using StellaOps.Scanner.Analyzers.Lang.Go;
 | 
			
		||||
using StellaOps.Scanner.Analyzers.Lang.Java;
 | 
			
		||||
using StellaOps.Scanner.Analyzers.Lang.Node;
 | 
			
		||||
using StellaOps.Scanner.Analyzers.Lang.DotNet;
 | 
			
		||||
using StellaOps.Scanner.Analyzers.Lang.Python;
 | 
			
		||||
 | 
			
		||||
namespace StellaOps.Bench.ScannerAnalyzers.Scenarios;
 | 
			
		||||
 | 
			
		||||
@@ -109,6 +110,7 @@ internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner
 | 
			
		||||
            "go" => static () => new GoLanguageAnalyzer(),
 | 
			
		||||
            "node" => static () => new NodeLanguageAnalyzer(),
 | 
			
		||||
            "dotnet" => static () => new DotNetLanguageAnalyzer(),
 | 
			
		||||
            "python" => static () => new PythonLanguageAnalyzer(),
 | 
			
		||||
            _ => throw new InvalidOperationException($"Unsupported analyzer '{analyzerId}'."),
 | 
			
		||||
        };
 | 
			
		||||
    }
 | 
			
		||||
 
 | 
			
		||||
@@ -14,5 +14,10 @@
 | 
			
		||||
    <ProjectReference Include="..\..\..\src\StellaOps.Scanner.Analyzers.Lang.Node\StellaOps.Scanner.Analyzers.Lang.Node.csproj" />
 | 
			
		||||
    <ProjectReference Include="..\..\..\src\StellaOps.Scanner.Analyzers.Lang.Java\StellaOps.Scanner.Analyzers.Lang.Java.csproj" />
 | 
			
		||||
    <ProjectReference Include="..\..\..\src\StellaOps.Scanner.Analyzers.Lang.DotNet\StellaOps.Scanner.Analyzers.Lang.DotNet.csproj" />
 | 
			
		||||
    <ProjectReference Include="..\..\..\src\StellaOps.Scanner.Analyzers.Lang.Python\StellaOps.Scanner.Analyzers.Lang.Python.csproj" />
 | 
			
		||||
  </ItemGroup>
 | 
			
		||||
 | 
			
		||||
  <ItemGroup>
 | 
			
		||||
    <InternalsVisibleTo Include="StellaOps.Bench.ScannerAnalyzers.Tests" />
 | 
			
		||||
  </ItemGroup>
 | 
			
		||||
</Project>
 | 
			
		||||
 
 | 
			
		||||
		Reference in New Issue
	
	Block a user