feat: Add RustFS artifact object store and migration tool
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS.
- Added unit tests for RustFsArtifactObjectStore functionality.
- Created a RustFS migrator tool to transfer objects from S3 to RustFS.
- Introduced policy preview and report models for API integration.
- Added fixtures and tests for policy preview and report functionality.
- Included necessary metadata and scripts for cache_pkg package.
This commit is contained in:
Vladimir Moushkov
2025-10-23 18:53:18 +03:00
parent aaa5fbfb78
commit f4d7a15a00
117 changed files with 4849 additions and 725 deletions

View File

@@ -0,0 +1,108 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using StellaOps.Bench.ScannerAnalyzers.Baseline;
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
internal static class BenchmarkJsonWriter
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
{
WriteIndented = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public static async Task WriteAsync(
string path,
BenchmarkJsonMetadata metadata,
IReadOnlyList<BenchmarkScenarioReport> reports,
CancellationToken cancellationToken)
{
ArgumentException.ThrowIfNullOrWhiteSpace(path);
ArgumentNullException.ThrowIfNull(metadata);
ArgumentNullException.ThrowIfNull(reports);
var resolved = Path.GetFullPath(path);
var directory = Path.GetDirectoryName(resolved);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
}
var document = new BenchmarkJsonDocument(
metadata.SchemaVersion,
metadata.CapturedAtUtc,
metadata.Commit,
metadata.Environment,
reports.Select(CreateScenario).ToArray());
await using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None);
await JsonSerializer.SerializeAsync(stream, document, SerializerOptions, cancellationToken).ConfigureAwait(false);
await stream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
private static BenchmarkJsonScenario CreateScenario(BenchmarkScenarioReport report)
{
var baseline = report.Baseline;
return new BenchmarkJsonScenario(
report.Result.Id,
report.Result.Label,
report.Result.Iterations,
report.Result.SampleCount,
report.Result.MeanMs,
report.Result.P95Ms,
report.Result.MaxMs,
report.Result.ThresholdMs,
baseline is null
? null
: new BenchmarkJsonScenarioBaseline(
baseline.Iterations,
baseline.SampleCount,
baseline.MeanMs,
baseline.P95Ms,
baseline.MaxMs),
new BenchmarkJsonScenarioRegression(
report.MaxRegressionRatio,
report.MeanRegressionRatio,
report.RegressionLimit,
report.RegressionBreached));
}
private sealed record BenchmarkJsonDocument(
string SchemaVersion,
DateTimeOffset CapturedAt,
string? Commit,
string? Environment,
IReadOnlyList<BenchmarkJsonScenario> Scenarios);
private sealed record BenchmarkJsonScenario(
string Id,
string Label,
int Iterations,
int SampleCount,
double MeanMs,
double P95Ms,
double MaxMs,
double ThresholdMs,
BenchmarkJsonScenarioBaseline? Baseline,
BenchmarkJsonScenarioRegression Regression);
private sealed record BenchmarkJsonScenarioBaseline(
int Iterations,
int SampleCount,
double MeanMs,
double P95Ms,
double MaxMs);
private sealed record BenchmarkJsonScenarioRegression(
double? MaxRatio,
double? MeanRatio,
double Limit,
bool Breached);
}
internal sealed record BenchmarkJsonMetadata(
string SchemaVersion,
DateTimeOffset CapturedAtUtc,
string? Commit,
string? Environment);

View File

@@ -0,0 +1,55 @@
using StellaOps.Bench.ScannerAnalyzers.Baseline;
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
internal sealed class BenchmarkScenarioReport
{
private const double RegressionLimitDefault = 1.2d;
public BenchmarkScenarioReport(ScenarioResult result, BaselineEntry? baseline, double? regressionLimit = null)
{
Result = result ?? throw new ArgumentNullException(nameof(result));
Baseline = baseline;
RegressionLimit = regressionLimit is { } limit && limit > 0 ? limit : RegressionLimitDefault;
MaxRegressionRatio = CalculateRatio(result.MaxMs, baseline?.MaxMs);
MeanRegressionRatio = CalculateRatio(result.MeanMs, baseline?.MeanMs);
}
public ScenarioResult Result { get; }
public BaselineEntry? Baseline { get; }
public double RegressionLimit { get; }
public double? MaxRegressionRatio { get; }
public double? MeanRegressionRatio { get; }
public bool RegressionBreached => MaxRegressionRatio.HasValue && MaxRegressionRatio.Value >= RegressionLimit;
public string? BuildRegressionFailureMessage()
{
if (!RegressionBreached || MaxRegressionRatio is null)
{
return null;
}
var percentage = (MaxRegressionRatio.Value - 1d) * 100d;
return $"{Result.Id} exceeded regression budget: max {Result.MaxMs:F2} ms vs baseline {Baseline!.MaxMs:F2} ms (+{percentage:F1}%)";
}
private static double? CalculateRatio(double current, double? baseline)
{
if (!baseline.HasValue)
{
return null;
}
if (baseline.Value <= 0d)
{
return null;
}
return current / baseline.Value;
}
}

View File

@@ -0,0 +1,59 @@
using System.Globalization;
using System.Text;
namespace StellaOps.Bench.ScannerAnalyzers.Reporting;
internal static class PrometheusWriter
{
public static void Write(string path, IReadOnlyList<BenchmarkScenarioReport> reports)
{
ArgumentException.ThrowIfNullOrWhiteSpace(path);
ArgumentNullException.ThrowIfNull(reports);
var resolved = Path.GetFullPath(path);
var directory = Path.GetDirectoryName(resolved);
if (!string.IsNullOrEmpty(directory))
{
Directory.CreateDirectory(directory);
}
var builder = new StringBuilder();
builder.AppendLine("# HELP scanner_analyzer_bench_duration_ms Analyzer benchmark duration metrics in milliseconds.");
builder.AppendLine("# TYPE scanner_analyzer_bench_duration_ms gauge");
foreach (var report in reports)
{
var scenarioLabel = Escape(report.Result.Id);
AppendMetric(builder, "scanner_analyzer_bench_mean_ms", scenarioLabel, report.Result.MeanMs);
AppendMetric(builder, "scanner_analyzer_bench_p95_ms", scenarioLabel, report.Result.P95Ms);
AppendMetric(builder, "scanner_analyzer_bench_max_ms", scenarioLabel, report.Result.MaxMs);
AppendMetric(builder, "scanner_analyzer_bench_threshold_ms", scenarioLabel, report.Result.ThresholdMs);
if (report.Baseline is { } baseline)
{
AppendMetric(builder, "scanner_analyzer_bench_baseline_max_ms", scenarioLabel, baseline.MaxMs);
AppendMetric(builder, "scanner_analyzer_bench_baseline_mean_ms", scenarioLabel, baseline.MeanMs);
}
if (report.MaxRegressionRatio is { } ratio)
{
AppendMetric(builder, "scanner_analyzer_bench_regression_ratio", scenarioLabel, ratio);
AppendMetric(builder, "scanner_analyzer_bench_regression_limit", scenarioLabel, report.RegressionLimit);
AppendMetric(builder, "scanner_analyzer_bench_regression_breached", scenarioLabel, report.RegressionBreached ? 1 : 0);
}
}
File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8);
}
private static void AppendMetric(StringBuilder builder, string metric, string scenarioLabel, double value)
{
builder.Append(metric);
builder.Append("{scenario=\"");
builder.Append(scenarioLabel);
builder.Append("\"} ");
builder.AppendLine(value.ToString("G17", CultureInfo.InvariantCulture));
}
private static string Escape(string value) => value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal);
}