git.stella-ops.org/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs

using System.Globalization;
using StellaOps.Bench.ScannerAnalyzers.Scenarios;

namespace StellaOps.Bench.ScannerAnalyzers;

internal static class Program
{
    public static async Task<int> Main(string[] args)
    {
        try
        {
            var options = ProgramOptions.Parse(args);
            var config = await BenchmarkConfig.LoadAsync(options.ConfigPath).ConfigureAwait(false);

            var iterations = options.Iterations ?? config.Iterations ?? 5;
            var thresholdMs = options.ThresholdMs ?? config.ThresholdMs ?? 5000;
            var repoRoot = ResolveRepoRoot(options.RepoRoot, options.ConfigPath);

            var results = new List<ScenarioResult>();
            var failures = new List<string>();

            foreach (var scenario in config.Scenarios)
            {
                var runner = ScenarioRunnerFactory.Create(scenario);
                var scenarioRoot = ResolveScenarioRoot(repoRoot, scenario.Root!);

                var execution = await runner.ExecuteAsync(scenarioRoot, iterations, CancellationToken.None).ConfigureAwait(false);
                var stats = ScenarioStatistics.FromDurations(execution.Durations);
                var scenarioThreshold = scenario.ThresholdMs ?? thresholdMs;

                results.Add(new ScenarioResult(
                    scenario.Id!,
                    scenario.Label ?? scenario.Id!,
                    execution.SampleCount,
                    stats.MeanMs,
                    stats.P95Ms,
                    stats.MaxMs,
                    iterations));

                if (stats.MaxMs > scenarioThreshold)
                {
                    failures.Add($"{scenario.Id} exceeded threshold: {stats.MaxMs:F2} ms > {scenarioThreshold:F2} ms");
                }
            }

            TablePrinter.Print(results);

            if (!string.IsNullOrWhiteSpace(options.OutPath))
            {
                CsvWriter.Write(options.OutPath!, results);
            }

            if (failures.Count > 0)
            {
                Console.Error.WriteLine();
                Console.Error.WriteLine("Performance threshold exceeded:");
                foreach (var failure in failures)
                {
                    Console.Error.WriteLine($" - {failure}");
                }

                return 1;
            }

            return 0;
        }
        catch (Exception ex)
        {
            Console.Error.WriteLine(ex.Message);
            return 1;
        }
    }

    private static string ResolveRepoRoot(string? overridePath, string configPath)
    {
        if (!string.IsNullOrWhiteSpace(overridePath))
        {
            return Path.GetFullPath(overridePath);
        }

        var configDirectory = Path.GetDirectoryName(configPath);
        if (string.IsNullOrWhiteSpace(configDirectory))
        {
            return Directory.GetCurrentDirectory();
        }

        return Path.GetFullPath(Path.Combine(configDirectory, "..", ".."));
    }

    private static string ResolveScenarioRoot(string repoRoot, string relativeRoot)
    {
        if (string.IsNullOrWhiteSpace(relativeRoot))
        {
            throw new InvalidOperationException("Scenario root is required.");
        }

        var combined = Path.GetFullPath(Path.Combine(repoRoot, relativeRoot));
        if (!PathUtilities.IsWithinRoot(repoRoot, combined))
        {
            throw new InvalidOperationException($"Scenario root '{relativeRoot}' escapes repository root '{repoRoot}'.");
        }

        if (!Directory.Exists(combined))
        {
            throw new DirectoryNotFoundException($"Scenario root '{combined}' does not exist.");
        }

        return combined;
    }

    private sealed record ProgramOptions(string ConfigPath, int? Iterations, double? ThresholdMs, string? OutPath, string? RepoRoot)
    {
        public static ProgramOptions Parse(string[] args)
        {
            var configPath = DefaultConfigPath();
            int? iterations = null;
            double? thresholdMs = null;
            string? outPath = null;
            string? repoRoot = null;

            for (var index = 0; index < args.Length; index++)
            {
                var current = args[index];
                switch (current)
                {
                    case "--config":
                        EnsureNext(args, index);
                        configPath = Path.GetFullPath(args[++index]);
                        break;
                    case "--iterations":
                        EnsureNext(args, index);
                        iterations = int.Parse(args[++index], CultureInfo.InvariantCulture);
                        break;
                    case "--threshold-ms":
                        EnsureNext(args, index);
                        thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture);
                        break;
                    case "--out":
                        EnsureNext(args, index);
                        outPath = args[++index];
                        break;
                    case "--repo-root":
                    case "--samples":
                        EnsureNext(args, index);
                        repoRoot = args[++index];
                        break;
                    default:
                        throw new ArgumentException($"Unknown argument: {current}", nameof(args));
                }
            }

            return new ProgramOptions(configPath, iterations, thresholdMs, outPath, repoRoot);
        }

        private static string DefaultConfigPath()
        {
            var binaryDir = AppContext.BaseDirectory;
            var projectRoot = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", ".."));
            var configDirectory = Path.GetFullPath(Path.Combine(projectRoot, ".."));
            return Path.Combine(configDirectory, "config.json");
        }

        private static void EnsureNext(string[] args, int index)
        {
            if (index + 1 >= args.Length)
            {
                throw new ArgumentException("Missing value for argument.", nameof(args));
            }
        }
    }

    private sealed record ScenarioResult(
        string Id,
        string Label,
        int SampleCount,
        double MeanMs,
        double P95Ms,
        double MaxMs,
        int Iterations);

    private sealed record ScenarioStatistics(double MeanMs, double P95Ms, double MaxMs)
    {
        public static ScenarioStatistics FromDurations(IReadOnlyList<double> durations)
        {
            if (durations.Count == 0)
            {
                return new ScenarioStatistics(0, 0, 0);
            }

            var sorted = durations.ToArray();
            Array.Sort(sorted);

            var total = 0d;
            foreach (var value in durations)
            {
                total += value;
            }

            var mean = total / durations.Count;
            var p95 = Percentile(sorted, 95);
            var max = sorted[^1];

            return new ScenarioStatistics(mean, p95, max);
        }

        private static double Percentile(IReadOnlyList<double> sorted, double percentile)
        {
            if (sorted.Count == 0)
            {
                return 0;
            }

            var rank = (percentile / 100d) * (sorted.Count - 1);
            var lower = (int)Math.Floor(rank);
            var upper = (int)Math.Ceiling(rank);
            var weight = rank - lower;

            if (upper >= sorted.Count)
            {
                return sorted[lower];
            }

            return sorted[lower] + weight * (sorted[upper] - sorted[lower]);
        }
    }

    private static class TablePrinter
    {
        public static void Print(IEnumerable<ScenarioResult> results)
        {
            Console.WriteLine("Scenario                     | Count |   Mean(ms) |    P95(ms) |     Max(ms)");
            Console.WriteLine("---------------------------- | ----- | --------- | --------- | ----------");
            foreach (var row in results)
            {
                Console.WriteLine(FormatRow(row));
            }
        }

        private static string FormatRow(ScenarioResult row)
        {
            var idColumn = row.Id.Length <= 28
                ? row.Id.PadRight(28)
                : row.Id[..28];

            return string.Join(" | ", new[]
            {
                idColumn,
                row.SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5),
                row.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9),
                row.P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9),
                row.MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10),
            });
        }
    }

    private static class CsvWriter
    {
        public static void Write(string path, IEnumerable<ScenarioResult> results)
        {
            var resolvedPath = Path.GetFullPath(path);
            var directory = Path.GetDirectoryName(resolvedPath);
            if (!string.IsNullOrEmpty(directory))
            {
                Directory.CreateDirectory(directory);
            }

            using var stream = new FileStream(resolvedPath, FileMode.Create, FileAccess.Write, FileShare.None);
            using var writer = new StreamWriter(stream);
            writer.WriteLine("scenario,iterations,sample_count,mean_ms,p95_ms,max_ms");

            foreach (var row in results)
            {
                writer.Write(row.Id);
                writer.Write(',');
                writer.Write(row.Iterations.ToString(CultureInfo.InvariantCulture));
                writer.Write(',');
                writer.Write(row.SampleCount.ToString(CultureInfo.InvariantCulture));
                writer.Write(',');
                writer.Write(row.MeanMs.ToString("F4", CultureInfo.InvariantCulture));
                writer.Write(',');
                writer.Write(row.P95Ms.ToString("F4", CultureInfo.InvariantCulture));
                writer.Write(',');
                writer.Write(row.MaxMs.ToString("F4", CultureInfo.InvariantCulture));
                writer.WriteLine();
            }
        }
    }

    internal static class PathUtilities
    {
        public static bool IsWithinRoot(string root, string candidate)
        {
            var relative = Path.GetRelativePath(root, candidate);
            if (string.IsNullOrEmpty(relative) || relative == ".")
            {
                return true;
            }

            return !relative.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relative);
        }
    }
}