feat: Add RustFS artifact object store and migration tool
- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS. - Added unit tests for RustFsArtifactObjectStore functionality. - Created a RustFS migrator tool to transfer objects from S3 to RustFS. - Introduced policy preview and report models for API integration. - Added fixtures and tests for policy preview and report functionality. - Included necessary metadata and scripts for cache_pkg package.
This commit is contained in:
master
@@ -19,7 +19,10 @@ dotnet run \
|
||||
--project bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj \
|
||||
-- \
|
||||
--repo-root . \
|
||||
--out bench/Scanner.Analyzers/baseline.csv
|
||||
--out bench/Scanner.Analyzers/baseline.csv \
|
||||
--json out/bench/scanner-analyzers/latest.json \
|
||||
--prom out/bench/scanner-analyzers/latest.prom \
|
||||
--commit "$(git rev-parse HEAD)"
|
||||
```
|
||||
|
||||
The harness prints a table to stdout and writes the CSV (if `--out` is specified) with the following headers:
|
||||
@@ -28,7 +31,16 @@ The harness prints a table to stdout and writes the CSV (if `--out` is specified
|
||||
scenario,iterations,sample_count,mean_ms,p95_ms,max_ms
|
||||
```
|
||||
|
||||
Use `--iterations` to override the default (5 passes per scenario) and `--threshold-ms` to customize the failure budget. Budgets default to 5 000 ms (or per-scenario overrides in `config.json`), aligned with the SBOM compose objective.
|
||||
Additional outputs:
|
||||
- `--json` emits a deterministic report consumable by Grafana/automation (schema `1.0`, see `docs/12_PERFORMANCE_WORKBOOK.md`).
|
||||
- `--prom` exports Prometheus-compatible gauges (`scanner_analyzer_bench_*`), which CI uploads for dashboards and alerts.
|
||||
|
||||
Use `--iterations` to override the default (5 passes per scenario) and `--threshold-ms` to customize the failure budget. Budgets default to 5 000 ms (or per-scenario overrides in `config.json`), aligned with the SBOM compose objective. Provide `--baseline path/to/baseline.csv` (defaults to the repo baseline) to compare against historical numbers—regressions ≥ 20 % on the `max_ms` metric or breaches of the configured threshold will fail the run.
|
||||
|
||||
Metadata options:
|
||||
- `--captured-at 2025-10-23T12:00:00Z` to inject a deterministic timestamp (otherwise `UtcNow`).
|
||||
- `--commit` and `--environment` annotate the JSON report for dashboards.
|
||||
- `--regression-limit 1.15` adjusts the ratio guard (default 1.20 ⇒ +20 %).
|
||||
|
||||
## Adding scenarios
|
||||
1. Drop the fixture tree under `samples/<area>/...`.
|
||||
@@ -38,5 +50,5 @@ Use `--iterations` to override the default (5 passes per scenario) and `--thresh
|
||||
- `root` – path to the directory that will be scanned.
|
||||
- For analyzer-backed scenarios, set `analyzers` to the list of language analyzer ids (for example, `["node"]`).
|
||||
- For temporary metadata walks (used until the analyzer ships), provide `parser` (`node` or `python`) and the `matcher` glob describing files to parse.
|
||||
3. Re-run the harness (`dotnet run … --out baseline.csv`).
|
||||
3. Re-run the harness (`dotnet run … --out baseline.csv --json out/.../new.json --prom out/.../new.prom`).
|
||||
4. Commit both the fixture and updated baseline.
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
using System.Text;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Tests;
|
||||
|
||||
public sealed class BaselineLoaderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task LoadAsync_ReadsCsvIntoDictionary()
|
||||
{
|
||||
var csv = """
|
||||
scenario,iterations,sample_count,mean_ms,p95_ms,max_ms
|
||||
node_monorepo_walk,5,4,9.4303,36.1354,45.0012
|
||||
python_site_packages_walk,5,10,12.1000,18.2000,26.3000
|
||||
""";
|
||||
|
||||
var path = await WriteTempFileAsync(csv);
|
||||
|
||||
var result = await BaselineLoader.LoadAsync(path, CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, result.Count);
|
||||
var entry = Assert.Contains("node_monorepo_walk", result);
|
||||
Assert.Equal(5, entry.Iterations);
|
||||
Assert.Equal(4, entry.SampleCount);
|
||||
Assert.Equal(9.4303, entry.MeanMs, 4);
|
||||
Assert.Equal(36.1354, entry.P95Ms, 4);
|
||||
Assert.Equal(45.0012, entry.MaxMs, 4);
|
||||
}
|
||||
|
||||
private static async Task<string> WriteTempFileAsync(string content)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), $"baseline-{Guid.NewGuid():N}.csv");
|
||||
await File.WriteAllTextAsync(path, content, Encoding.UTF8);
|
||||
return path;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using System.Text.Json;
|
||||
using StellaOps.Bench.ScannerAnalyzers;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Reporting;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Tests;
|
||||
|
||||
public sealed class BenchmarkJsonWriterTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task WriteAsync_EmitsMetadataAndScenarioDetails()
|
||||
{
|
||||
var metadata = new BenchmarkJsonMetadata("1.0", DateTimeOffset.Parse("2025-10-23T12:00:00Z"), "abc123", "ci");
|
||||
var result = new ScenarioResult(
|
||||
"scenario",
|
||||
"Scenario",
|
||||
SampleCount: 5,
|
||||
MeanMs: 10,
|
||||
P95Ms: 12,
|
||||
MaxMs: 20,
|
||||
Iterations: 5,
|
||||
ThresholdMs: 5000);
|
||||
var baseline = new BaselineEntry("scenario", 5, 5, 9, 11, 10);
|
||||
var report = new BenchmarkScenarioReport(result, baseline, 1.2);
|
||||
|
||||
var path = Path.Combine(Path.GetTempPath(), $"bench-{Guid.NewGuid():N}.json");
|
||||
await BenchmarkJsonWriter.WriteAsync(path, metadata, new[] { report }, CancellationToken.None);
|
||||
|
||||
using var document = JsonDocument.Parse(await File.ReadAllTextAsync(path));
|
||||
var root = document.RootElement;
|
||||
|
||||
Assert.Equal("1.0", root.GetProperty("schemaVersion").GetString());
|
||||
Assert.Equal("abc123", root.GetProperty("commit").GetString());
|
||||
var scenario = root.GetProperty("scenarios")[0];
|
||||
Assert.Equal("scenario", scenario.GetProperty("id").GetString());
|
||||
Assert.Equal(20, scenario.GetProperty("maxMs").GetDouble());
|
||||
Assert.Equal(10, scenario.GetProperty("baseline").GetProperty("maxMs").GetDouble());
|
||||
Assert.True(scenario.GetProperty("regression").GetProperty("breached").GetBoolean());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using StellaOps.Bench.ScannerAnalyzers;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Reporting;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Tests;
|
||||
|
||||
public sealed class BenchmarkScenarioReportTests
|
||||
{
|
||||
[Fact]
|
||||
public void RegressionRatio_ComputedWhenBaselinePresent()
|
||||
{
|
||||
var result = new ScenarioResult(
|
||||
"scenario",
|
||||
"Scenario",
|
||||
SampleCount: 5,
|
||||
MeanMs: 10,
|
||||
P95Ms: 12,
|
||||
MaxMs: 20,
|
||||
Iterations: 5,
|
||||
ThresholdMs: 5000);
|
||||
|
||||
var baseline = new BaselineEntry(
|
||||
"scenario",
|
||||
Iterations: 5,
|
||||
SampleCount: 5,
|
||||
MeanMs: 8,
|
||||
P95Ms: 11,
|
||||
MaxMs: 15);
|
||||
|
||||
var report = new BenchmarkScenarioReport(result, baseline, regressionLimit: 1.2);
|
||||
|
||||
Assert.True(report.MaxRegressionRatio.HasValue);
|
||||
Assert.Equal(20d / 15d, report.MaxRegressionRatio.Value, 6);
|
||||
Assert.True(report.RegressionBreached);
|
||||
Assert.Contains("+33.3%", report.BuildRegressionFailureMessage());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RegressionRatio_NullWhenBaselineMissing()
|
||||
{
|
||||
var result = new ScenarioResult(
|
||||
"scenario",
|
||||
"Scenario",
|
||||
SampleCount: 5,
|
||||
MeanMs: 10,
|
||||
P95Ms: 12,
|
||||
MaxMs: 20,
|
||||
Iterations: 5,
|
||||
ThresholdMs: 5000);
|
||||
|
||||
var report = new BenchmarkScenarioReport(result, baseline: null, regressionLimit: 1.2);
|
||||
|
||||
Assert.Null(report.MaxRegressionRatio);
|
||||
Assert.False(report.RegressionBreached);
|
||||
Assert.Null(report.BuildRegressionFailureMessage());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
using StellaOps.Bench.ScannerAnalyzers;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Reporting;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Tests;
|
||||
|
||||
public sealed class PrometheusWriterTests
|
||||
{
|
||||
[Fact]
|
||||
public void Write_EmitsMetricsForScenario()
|
||||
{
|
||||
var result = new ScenarioResult(
|
||||
"scenario_a",
|
||||
"Scenario A",
|
||||
SampleCount: 5,
|
||||
MeanMs: 10,
|
||||
P95Ms: 12,
|
||||
MaxMs: 20,
|
||||
Iterations: 5,
|
||||
ThresholdMs: 5000);
|
||||
var baseline = new BaselineEntry("scenario_a", 5, 5, 9, 11, 18);
|
||||
var report = new BenchmarkScenarioReport(result, baseline, 1.2);
|
||||
|
||||
var path = Path.Combine(Path.GetTempPath(), $"metrics-{Guid.NewGuid():N}.prom");
|
||||
PrometheusWriter.Write(path, new[] { report });
|
||||
|
||||
var contents = File.ReadAllText(path);
|
||||
Assert.Contains("scanner_analyzer_bench_max_ms{scenario=\"scenario_a\"} 20", contents);
|
||||
Assert.Contains("scanner_analyzer_bench_regression_ratio{scenario=\"scenario_a\"}", contents);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Bench.ScannerAnalyzers\StellaOps.Bench.ScannerAnalyzers.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
|
||||
internal sealed record BaselineEntry(
|
||||
string ScenarioId,
|
||||
int Iterations,
|
||||
int SampleCount,
|
||||
double MeanMs,
|
||||
double P95Ms,
|
||||
double MaxMs);
|
||||
@@ -0,0 +1,88 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
|
||||
internal static class BaselineLoader
|
||||
{
|
||||
public static async Task<IReadOnlyDictionary<string, BaselineEntry>> LoadAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
throw new ArgumentException("Baseline path must be provided.", nameof(path));
|
||||
}
|
||||
|
||||
var resolved = Path.GetFullPath(path);
|
||||
if (!File.Exists(resolved))
|
||||
{
|
||||
throw new FileNotFoundException($"Baseline file not found at {resolved}", resolved);
|
||||
}
|
||||
|
||||
var result = new Dictionary<string, BaselineEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
await using var stream = new FileStream(resolved, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var reader = new StreamReader(stream);
|
||||
string? line;
|
||||
var isFirst = true;
|
||||
|
||||
while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isFirst)
|
||||
{
|
||||
isFirst = false;
|
||||
if (line.StartsWith("scenario,", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
var entry = ParseLine(line);
|
||||
result[entry.ScenarioId] = entry;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static BaselineEntry ParseLine(string line)
|
||||
{
|
||||
var parts = line.Split(',', StringSplitOptions.TrimEntries);
|
||||
if (parts.Length < 6)
|
||||
{
|
||||
throw new InvalidDataException($"Baseline CSV row malformed: '{line}'");
|
||||
}
|
||||
|
||||
var scenarioId = parts[0];
|
||||
var iterations = ParseInt(parts[1], nameof(BaselineEntry.Iterations));
|
||||
var sampleCount = ParseInt(parts[2], nameof(BaselineEntry.SampleCount));
|
||||
var meanMs = ParseDouble(parts[3], nameof(BaselineEntry.MeanMs));
|
||||
var p95Ms = ParseDouble(parts[4], nameof(BaselineEntry.P95Ms));
|
||||
var maxMs = ParseDouble(parts[5], nameof(BaselineEntry.MaxMs));
|
||||
|
||||
return new BaselineEntry(scenarioId, iterations, sampleCount, meanMs, p95Ms, maxMs);
|
||||
}
|
||||
|
||||
private static int ParseInt(string value, string field)
|
||||
{
|
||||
if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
throw new InvalidDataException($"Failed to parse integer {field} from '{value}'.");
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
private static double ParseDouble(string value, string field)
|
||||
{
|
||||
if (!double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed))
|
||||
{
|
||||
throw new InvalidDataException($"Failed to parse double {field} from '{value}'.");
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
using System.Globalization;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Reporting;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Scenarios;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers;
|
||||
@@ -15,8 +17,13 @@ internal static class Program
|
||||
var iterations = options.Iterations ?? config.Iterations ?? 5;
|
||||
var thresholdMs = options.ThresholdMs ?? config.ThresholdMs ?? 5000;
|
||||
var repoRoot = ResolveRepoRoot(options.RepoRoot, options.ConfigPath);
|
||||
var regressionLimit = options.RegressionLimit ?? 1.2d;
|
||||
var capturedAt = (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime();
|
||||
|
||||
var baseline = await LoadBaselineDictionaryAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
var results = new List<ScenarioResult>();
|
||||
var reports = new List<BenchmarkScenarioReport>();
|
||||
var failures = new List<string>();
|
||||
|
||||
foreach (var scenario in config.Scenarios)
|
||||
@@ -28,26 +35,54 @@ internal static class Program
|
||||
var stats = ScenarioStatistics.FromDurations(execution.Durations);
|
||||
var scenarioThreshold = scenario.ThresholdMs ?? thresholdMs;
|
||||
|
||||
results.Add(new ScenarioResult(
|
||||
var result = new ScenarioResult(
|
||||
scenario.Id!,
|
||||
scenario.Label ?? scenario.Id!,
|
||||
execution.SampleCount,
|
||||
stats.MeanMs,
|
||||
stats.P95Ms,
|
||||
stats.MaxMs,
|
||||
iterations));
|
||||
iterations,
|
||||
scenarioThreshold);
|
||||
|
||||
results.Add(result);
|
||||
|
||||
if (stats.MaxMs > scenarioThreshold)
|
||||
{
|
||||
failures.Add($"{scenario.Id} exceeded threshold: {stats.MaxMs:F2} ms > {scenarioThreshold:F2} ms");
|
||||
}
|
||||
|
||||
baseline.TryGetValue(result.Id, out var baselineEntry);
|
||||
var report = new BenchmarkScenarioReport(result, baselineEntry, regressionLimit);
|
||||
if (report.BuildRegressionFailureMessage() is { } regressionFailure)
|
||||
{
|
||||
failures.Add(regressionFailure);
|
||||
}
|
||||
|
||||
reports.Add(report);
|
||||
}
|
||||
|
||||
TablePrinter.Print(results);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.OutPath))
|
||||
if (!string.IsNullOrWhiteSpace(options.CsvOutPath))
|
||||
{
|
||||
CsvWriter.Write(options.OutPath!, results);
|
||||
CsvWriter.Write(options.CsvOutPath!, results);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.JsonOutPath))
|
||||
{
|
||||
var metadata = new BenchmarkJsonMetadata(
|
||||
"1.0",
|
||||
capturedAt,
|
||||
options.Commit,
|
||||
options.Environment);
|
||||
|
||||
await BenchmarkJsonWriter.WriteAsync(options.JsonOutPath!, metadata, reports, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath))
|
||||
{
|
||||
PrometheusWriter.Write(options.PrometheusOutPath!, reports);
|
||||
}
|
||||
|
||||
if (failures.Count > 0)
|
||||
@@ -71,6 +106,22 @@ internal static class Program
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyDictionary<string, BaselineEntry>> LoadBaselineDictionaryAsync(string? baselinePath, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(baselinePath))
|
||||
{
|
||||
return new Dictionary<string, BaselineEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
var resolved = Path.GetFullPath(baselinePath);
|
||||
if (!File.Exists(resolved))
|
||||
{
|
||||
return new Dictionary<string, BaselineEntry>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
return await BaselineLoader.LoadAsync(resolved, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static string ResolveRepoRoot(string? overridePath, string configPath)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(overridePath))
|
||||
@@ -108,15 +159,34 @@ internal static class Program
|
||||
return combined;
|
||||
}
|
||||
|
||||
private sealed record ProgramOptions(string ConfigPath, int? Iterations, double? ThresholdMs, string? OutPath, string? RepoRoot)
|
||||
private sealed record ProgramOptions(
|
||||
string ConfigPath,
|
||||
int? Iterations,
|
||||
double? ThresholdMs,
|
||||
string? CsvOutPath,
|
||||
string? JsonOutPath,
|
||||
string? PrometheusOutPath,
|
||||
string? RepoRoot,
|
||||
string? BaselinePath,
|
||||
DateTimeOffset? CapturedAtUtc,
|
||||
string? Commit,
|
||||
string? Environment,
|
||||
double? RegressionLimit)
|
||||
{
|
||||
public static ProgramOptions Parse(string[] args)
|
||||
{
|
||||
var configPath = DefaultConfigPath();
|
||||
var baselinePath = DefaultBaselinePath();
|
||||
int? iterations = null;
|
||||
double? thresholdMs = null;
|
||||
string? outPath = null;
|
||||
string? csvOut = null;
|
||||
string? jsonOut = null;
|
||||
string? promOut = null;
|
||||
string? repoRoot = null;
|
||||
DateTimeOffset? capturedAt = null;
|
||||
string? commit = null;
|
||||
string? environment = null;
|
||||
double? regressionLimit = null;
|
||||
|
||||
for (var index = 0; index < args.Length; index++)
|
||||
{
|
||||
@@ -136,20 +206,50 @@ internal static class Program
|
||||
thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
||||
break;
|
||||
case "--out":
|
||||
case "--csv":
|
||||
EnsureNext(args, index);
|
||||
outPath = args[++index];
|
||||
csvOut = args[++index];
|
||||
break;
|
||||
case "--json":
|
||||
EnsureNext(args, index);
|
||||
jsonOut = args[++index];
|
||||
break;
|
||||
case "--prom":
|
||||
case "--prometheus":
|
||||
EnsureNext(args, index);
|
||||
promOut = args[++index];
|
||||
break;
|
||||
case "--baseline":
|
||||
EnsureNext(args, index);
|
||||
baselinePath = args[++index];
|
||||
break;
|
||||
case "--repo-root":
|
||||
case "--samples":
|
||||
EnsureNext(args, index);
|
||||
repoRoot = args[++index];
|
||||
break;
|
||||
case "--captured-at":
|
||||
EnsureNext(args, index);
|
||||
capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
|
||||
break;
|
||||
case "--commit":
|
||||
EnsureNext(args, index);
|
||||
commit = args[++index];
|
||||
break;
|
||||
case "--environment":
|
||||
EnsureNext(args, index);
|
||||
environment = args[++index];
|
||||
break;
|
||||
case "--regression-limit":
|
||||
EnsureNext(args, index);
|
||||
regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture);
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentException($"Unknown argument: {current}", nameof(args));
|
||||
}
|
||||
}
|
||||
|
||||
return new ProgramOptions(configPath, iterations, thresholdMs, outPath, repoRoot);
|
||||
return new ProgramOptions(configPath, iterations, thresholdMs, csvOut, jsonOut, promOut, repoRoot, baselinePath, capturedAt, commit, environment, regressionLimit);
|
||||
}
|
||||
|
||||
private static string DefaultConfigPath()
|
||||
@@ -160,6 +260,15 @@ internal static class Program
|
||||
return Path.Combine(configDirectory, "config.json");
|
||||
}
|
||||
|
||||
private static string? DefaultBaselinePath()
|
||||
{
|
||||
var binaryDir = AppContext.BaseDirectory;
|
||||
var projectRoot = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", ".."));
|
||||
var benchRoot = Path.GetFullPath(Path.Combine(projectRoot, ".."));
|
||||
var baselinePath = Path.Combine(benchRoot, "baseline.csv");
|
||||
return File.Exists(baselinePath) ? baselinePath : baselinePath;
|
||||
}
|
||||
|
||||
private static void EnsureNext(string[] args, int index)
|
||||
{
|
||||
if (index + 1 >= args.Length)
|
||||
@@ -169,15 +278,6 @@ internal static class Program
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record ScenarioResult(
|
||||
string Id,
|
||||
string Label,
|
||||
int SampleCount,
|
||||
double MeanMs,
|
||||
double P95Ms,
|
||||
double MaxMs,
|
||||
int Iterations);
|
||||
|
||||
private sealed record ScenarioStatistics(double MeanMs, double P95Ms, double MaxMs)
|
||||
{
|
||||
public static ScenarioStatistics FromDurations(IReadOnlyList<double> durations)
|
||||
@@ -232,25 +332,16 @@ internal static class Program
|
||||
Console.WriteLine("---------------------------- | ----- | --------- | --------- | ----------");
|
||||
foreach (var row in results)
|
||||
{
|
||||
Console.WriteLine(FormatRow(row));
|
||||
Console.WriteLine(string.Join(" | ", new[]
|
||||
{
|
||||
row.IdColumn,
|
||||
row.SampleCountColumn,
|
||||
row.MeanColumn,
|
||||
row.P95Column,
|
||||
row.MaxColumn
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
private static string FormatRow(ScenarioResult row)
|
||||
{
|
||||
var idColumn = row.Id.Length <= 28
|
||||
? row.Id.PadRight(28)
|
||||
: row.Id[..28];
|
||||
|
||||
return string.Join(" | ", new[]
|
||||
{
|
||||
idColumn,
|
||||
row.SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5),
|
||||
row.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9),
|
||||
row.P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9),
|
||||
row.MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static class CsvWriter
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
|
||||
|
||||
internal static class BenchmarkJsonWriter
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
public static async Task WriteAsync(
|
||||
string path,
|
||||
BenchmarkJsonMetadata metadata,
|
||||
IReadOnlyList<BenchmarkScenarioReport> reports,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(path);
|
||||
ArgumentNullException.ThrowIfNull(metadata);
|
||||
ArgumentNullException.ThrowIfNull(reports);
|
||||
|
||||
var resolved = Path.GetFullPath(path);
|
||||
var directory = Path.GetDirectoryName(resolved);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
var document = new BenchmarkJsonDocument(
|
||||
metadata.SchemaVersion,
|
||||
metadata.CapturedAtUtc,
|
||||
metadata.Commit,
|
||||
metadata.Environment,
|
||||
reports.Select(CreateScenario).ToArray());
|
||||
|
||||
await using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
await JsonSerializer.SerializeAsync(stream, document, SerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static BenchmarkJsonScenario CreateScenario(BenchmarkScenarioReport report)
|
||||
{
|
||||
var baseline = report.Baseline;
|
||||
return new BenchmarkJsonScenario(
|
||||
report.Result.Id,
|
||||
report.Result.Label,
|
||||
report.Result.Iterations,
|
||||
report.Result.SampleCount,
|
||||
report.Result.MeanMs,
|
||||
report.Result.P95Ms,
|
||||
report.Result.MaxMs,
|
||||
report.Result.ThresholdMs,
|
||||
baseline is null
|
||||
? null
|
||||
: new BenchmarkJsonScenarioBaseline(
|
||||
baseline.Iterations,
|
||||
baseline.SampleCount,
|
||||
baseline.MeanMs,
|
||||
baseline.P95Ms,
|
||||
baseline.MaxMs),
|
||||
new BenchmarkJsonScenarioRegression(
|
||||
report.MaxRegressionRatio,
|
||||
report.MeanRegressionRatio,
|
||||
report.RegressionLimit,
|
||||
report.RegressionBreached));
|
||||
}
|
||||
|
||||
private sealed record BenchmarkJsonDocument(
|
||||
string SchemaVersion,
|
||||
DateTimeOffset CapturedAt,
|
||||
string? Commit,
|
||||
string? Environment,
|
||||
IReadOnlyList<BenchmarkJsonScenario> Scenarios);
|
||||
|
||||
private sealed record BenchmarkJsonScenario(
|
||||
string Id,
|
||||
string Label,
|
||||
int Iterations,
|
||||
int SampleCount,
|
||||
double MeanMs,
|
||||
double P95Ms,
|
||||
double MaxMs,
|
||||
double ThresholdMs,
|
||||
BenchmarkJsonScenarioBaseline? Baseline,
|
||||
BenchmarkJsonScenarioRegression Regression);
|
||||
|
||||
private sealed record BenchmarkJsonScenarioBaseline(
|
||||
int Iterations,
|
||||
int SampleCount,
|
||||
double MeanMs,
|
||||
double P95Ms,
|
||||
double MaxMs);
|
||||
|
||||
private sealed record BenchmarkJsonScenarioRegression(
|
||||
double? MaxRatio,
|
||||
double? MeanRatio,
|
||||
double Limit,
|
||||
bool Breached);
|
||||
}
|
||||
|
||||
internal sealed record BenchmarkJsonMetadata(
|
||||
string SchemaVersion,
|
||||
DateTimeOffset CapturedAtUtc,
|
||||
string? Commit,
|
||||
string? Environment);
|
||||
@@ -0,0 +1,55 @@
|
||||
using StellaOps.Bench.ScannerAnalyzers.Baseline;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
|
||||
|
||||
internal sealed class BenchmarkScenarioReport
|
||||
{
|
||||
private const double RegressionLimitDefault = 1.2d;
|
||||
|
||||
public BenchmarkScenarioReport(ScenarioResult result, BaselineEntry? baseline, double? regressionLimit = null)
|
||||
{
|
||||
Result = result ?? throw new ArgumentNullException(nameof(result));
|
||||
Baseline = baseline;
|
||||
RegressionLimit = regressionLimit is { } limit && limit > 0 ? limit : RegressionLimitDefault;
|
||||
MaxRegressionRatio = CalculateRatio(result.MaxMs, baseline?.MaxMs);
|
||||
MeanRegressionRatio = CalculateRatio(result.MeanMs, baseline?.MeanMs);
|
||||
}
|
||||
|
||||
public ScenarioResult Result { get; }
|
||||
|
||||
public BaselineEntry? Baseline { get; }
|
||||
|
||||
public double RegressionLimit { get; }
|
||||
|
||||
public double? MaxRegressionRatio { get; }
|
||||
|
||||
public double? MeanRegressionRatio { get; }
|
||||
|
||||
public bool RegressionBreached => MaxRegressionRatio.HasValue && MaxRegressionRatio.Value >= RegressionLimit;
|
||||
|
||||
public string? BuildRegressionFailureMessage()
|
||||
{
|
||||
if (!RegressionBreached || MaxRegressionRatio is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var percentage = (MaxRegressionRatio.Value - 1d) * 100d;
|
||||
return $"{Result.Id} exceeded regression budget: max {Result.MaxMs:F2} ms vs baseline {Baseline!.MaxMs:F2} ms (+{percentage:F1}%)";
|
||||
}
|
||||
|
||||
private static double? CalculateRatio(double current, double? baseline)
|
||||
{
|
||||
if (!baseline.HasValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (baseline.Value <= 0d)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return current / baseline.Value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
|
||||
|
||||
internal static class PrometheusWriter
|
||||
{
|
||||
public static void Write(string path, IReadOnlyList<BenchmarkScenarioReport> reports)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(path);
|
||||
ArgumentNullException.ThrowIfNull(reports);
|
||||
|
||||
var resolved = Path.GetFullPath(path);
|
||||
var directory = Path.GetDirectoryName(resolved);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
var builder = new StringBuilder();
|
||||
builder.AppendLine("# HELP scanner_analyzer_bench_duration_ms Analyzer benchmark duration metrics in milliseconds.");
|
||||
builder.AppendLine("# TYPE scanner_analyzer_bench_duration_ms gauge");
|
||||
|
||||
foreach (var report in reports)
|
||||
{
|
||||
var scenarioLabel = Escape(report.Result.Id);
|
||||
AppendMetric(builder, "scanner_analyzer_bench_mean_ms", scenarioLabel, report.Result.MeanMs);
|
||||
AppendMetric(builder, "scanner_analyzer_bench_p95_ms", scenarioLabel, report.Result.P95Ms);
|
||||
AppendMetric(builder, "scanner_analyzer_bench_max_ms", scenarioLabel, report.Result.MaxMs);
|
||||
AppendMetric(builder, "scanner_analyzer_bench_threshold_ms", scenarioLabel, report.Result.ThresholdMs);
|
||||
|
||||
if (report.Baseline is { } baseline)
|
||||
{
|
||||
AppendMetric(builder, "scanner_analyzer_bench_baseline_max_ms", scenarioLabel, baseline.MaxMs);
|
||||
AppendMetric(builder, "scanner_analyzer_bench_baseline_mean_ms", scenarioLabel, baseline.MeanMs);
|
||||
}
|
||||
|
||||
if (report.MaxRegressionRatio is { } ratio)
|
||||
{
|
||||
AppendMetric(builder, "scanner_analyzer_bench_regression_ratio", scenarioLabel, ratio);
|
||||
AppendMetric(builder, "scanner_analyzer_bench_regression_limit", scenarioLabel, report.RegressionLimit);
|
||||
AppendMetric(builder, "scanner_analyzer_bench_regression_breached", scenarioLabel, report.RegressionBreached ? 1 : 0);
|
||||
}
|
||||
}
|
||||
|
||||
File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8);
|
||||
}
|
||||
|
||||
private static void AppendMetric(StringBuilder builder, string metric, string scenarioLabel, double value)
|
||||
{
|
||||
builder.Append(metric);
|
||||
builder.Append("{scenario=\"");
|
||||
builder.Append(scenarioLabel);
|
||||
builder.Append("\"} ");
|
||||
builder.AppendLine(value.ToString("G17", CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
private static string Escape(string value) => value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers;
|
||||
|
||||
internal sealed record ScenarioResult(
|
||||
string Id,
|
||||
string Label,
|
||||
int SampleCount,
|
||||
double MeanMs,
|
||||
double P95Ms,
|
||||
double MaxMs,
|
||||
int Iterations,
|
||||
double ThresholdMs)
|
||||
{
|
||||
public string IdColumn => Id.Length <= 28 ? Id.PadRight(28) : Id[..28];
|
||||
|
||||
public string SampleCountColumn => SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5);
|
||||
|
||||
public string MeanColumn => MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9);
|
||||
|
||||
public string P95Column => P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9);
|
||||
|
||||
public string MaxColumn => MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10);
|
||||
}
|
||||
@@ -8,6 +8,7 @@ using StellaOps.Scanner.Analyzers.Lang.Go;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Java;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Node;
|
||||
using StellaOps.Scanner.Analyzers.Lang.DotNet;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Python;
|
||||
|
||||
namespace StellaOps.Bench.ScannerAnalyzers.Scenarios;
|
||||
|
||||
@@ -109,6 +110,7 @@ internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner
|
||||
"go" => static () => new GoLanguageAnalyzer(),
|
||||
"node" => static () => new NodeLanguageAnalyzer(),
|
||||
"dotnet" => static () => new DotNetLanguageAnalyzer(),
|
||||
"python" => static () => new PythonLanguageAnalyzer(),
|
||||
_ => throw new InvalidOperationException($"Unsupported analyzer '{analyzerId}'."),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,5 +14,10 @@
|
||||
<ProjectReference Include="..\..\..\src\StellaOps.Scanner.Analyzers.Lang.Node\StellaOps.Scanner.Analyzers.Lang.Node.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\StellaOps.Scanner.Analyzers.Lang.Java\StellaOps.Scanner.Analyzers.Lang.Java.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\StellaOps.Scanner.Analyzers.Lang.DotNet\StellaOps.Scanner.Analyzers.Lang.DotNet.csproj" />
|
||||
<ProjectReference Include="..\..\..\src\StellaOps.Scanner.Analyzers.Lang.Python\StellaOps.Scanner.Analyzers.Lang.Python.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="StellaOps.Bench.ScannerAnalyzers.Tests" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
scenario,iterations,sample_count,mean_ms,p95_ms,max_ms
|
||||
node_monorepo_walk,5,4,9.4303,36.1354,45.0012
|
||||
java_demo_archive,5,1,20.6964,81.5592,101.7846
|
||||
go_buildinfo_fixture,5,2,35.0345,136.5466,170.1612
|
||||
dotnet_multirid_fixture,5,2,29.1862,106.6249,132.3018
|
||||
python_site_packages_walk,5,3,12.0024,45.0165,56.0003
|
||||
node_monorepo_walk,5,4,6.0975,21.7421,26.8537
|
||||
java_demo_archive,5,1,6.2007,23.4837,29.1143
|
||||
go_buildinfo_fixture,5,2,6.1949,22.6851,27.9196
|
||||
dotnet_multirid_fixture,5,2,11.4884,37.7460,46.4850
|
||||
python_site_packages_scan,5,3,5.6420,18.2943,22.3739
|
||||
python_pip_cache_fixture,5,1,5.8598,13.2855,15.6256
|
||||
|
||||
|
@@ -35,11 +35,20 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "python_site_packages_walk",
|
||||
"label": "Python site-packages dist-info crawl",
|
||||
"root": "samples/runtime/python-venv/lib/python3.11/site-packages",
|
||||
"matcher": "**/*.dist-info/METADATA",
|
||||
"parser": "python"
|
||||
}
|
||||
]
|
||||
}
|
||||
"id": "python_site_packages_scan",
|
||||
"label": "Python analyzer on sample virtualenv",
|
||||
"root": "samples/runtime/python-venv",
|
||||
"analyzers": [
|
||||
"python"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "python_pip_cache_fixture",
|
||||
"label": "Python analyzer verifying RECORD hashes",
|
||||
"root": "src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache",
|
||||
"analyzers": [
|
||||
"python"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -23,3 +23,9 @@ Results should be committed as deterministic CSV/JSON outputs with accompanying
|
||||
- Scenario `dotnet_multirid_fixture` exercises the .NET analyzer against the multi-RID test fixture that merges two applications and four runtime identifiers. Latest baseline run (Release build, 5 iterations) records a mean duration of **29.19 ms** (p95 106.62 ms, max 132.30 ms) with a stable component count of 2.
|
||||
- Syft v1.29.1 scanning the same fixture (`syft scan dir:…`) averaged **1 546 ms** (p95 ≈2 100 ms, max ≈2 100 ms) while also reporting duplicate packages; raw numbers captured in `dotnet/syft-comparison-20251023.csv`.
|
||||
- The new scenario is declared in `bench/Scanner.Analyzers/config.json`; rerun the bench command above after rebuilding analyzers to refresh baselines and comparison data.
|
||||
|
||||
## Sprint LA2 — Python Analyzer Benchmark Notes (2025-10-23)
|
||||
|
||||
- Added two Python scenarios to `config.json`: the virtualenv sample (`python_site_packages_scan`) and the RECORD-heavy pip cache fixture (`python_pip_cache_fixture`).
|
||||
- Baseline run (Release build, 5 iterations) records means of **5.64 ms** (p95 18.29 ms) for the virtualenv and **5.86 ms** (p95 13.29 ms) for the pip cache verifier; raw numbers stored in `python/hash-throughput-20251023.csv`.
|
||||
- The pip cache fixture exercises `PythonRecordVerifier` with 12 RECORD rows (7 hashed) and mismatched layer coverage, giving a repeatable hash-validation throughput reference for regression gating.
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
scenario,iterations,sample_count,mean_ms,p95_ms,max_ms
|
||||
python_site_packages_scan,5,3,5.6420,18.2943,22.3739
|
||||
python_pip_cache_fixture,5,1,5.8598,13.2855,15.6256
|
||||
|
Reference in New Issue
Block a user