git.stella-ops.org/tools/RustFsMigrator/Program.cs

using Amazon;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using System.Net.Http.Headers;

var options = MigrationOptions.Parse(args);
if (options is null)
{
    MigrationOptions.PrintUsage();
    return 1;
}

Console.WriteLine($"RustFS migrator starting (prefix: '{options.Prefix ?? "<all>"}')");
if (options.DryRun)
{
    Console.WriteLine("Dry-run enabled. No objects will be written to RustFS.");
}

var s3Config = new AmazonS3Config
{
    ForcePathStyle = true,
};

if (!string.IsNullOrWhiteSpace(options.S3ServiceUrl))
{
    s3Config.ServiceURL = options.S3ServiceUrl;
    s3Config.UseHttp = options.S3ServiceUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase);
}

if (!string.IsNullOrWhiteSpace(options.S3Region))
{
    s3Config.RegionEndpoint = RegionEndpoint.GetBySystemName(options.S3Region);
}

using var s3Client = CreateS3Client(options, s3Config);
using var httpClient = CreateRustFsClient(options);

var listRequest = new ListObjectsV2Request
{
    BucketName = options.S3Bucket,
    Prefix = options.Prefix,
    MaxKeys = 1000,
};

var migrated = 0;
var skipped = 0;

do
{
    var response = await s3Client.ListObjectsV2Async(listRequest).ConfigureAwait(false);
    foreach (var entry in response.S3Objects)
    {
        if (entry.Size == 0 && entry.Key.EndsWith('/'))
        {
            skipped++;
            continue;
        }

        Console.WriteLine($"Migrating {entry.Key} ({entry.Size} bytes)...");

        if (options.DryRun)
        {
            migrated++;
            continue;
        }

        using var getResponse = await s3Client.GetObjectAsync(new GetObjectRequest
        {
            BucketName = options.S3Bucket,
            Key = entry.Key,
        }).ConfigureAwait(false);

        await using var memory = new MemoryStream();
        await getResponse.ResponseStream.CopyToAsync(memory).ConfigureAwait(false);
        memory.Position = 0;

        using var request = new HttpRequestMessage(HttpMethod.Put, BuildRustFsUri(options, entry.Key))
        {
            Content = new ByteArrayContent(memory.ToArray()),
        };
        request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream");

        if (options.Immutable)
        {
            request.Headers.TryAddWithoutValidation("X-RustFS-Immutable", "true");
        }

        if (options.RetentionSeconds is { } retainSeconds)
        {
            request.Headers.TryAddWithoutValidation("X-RustFS-Retain-Seconds", retainSeconds.ToString());
        }

        if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
        {
            request.Headers.TryAddWithoutValidation(options.RustFsApiKeyHeader!, options.RustFsApiKey!);
        }

        using var responseMessage = await httpClient.SendAsync(request).ConfigureAwait(false);
        if (!responseMessage.IsSuccessStatusCode)
        {
            var error = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
            Console.Error.WriteLine($"Failed to upload {entry.Key}: {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}\n{error}");
            return 2;
        }

        migrated++;
    }

    listRequest.ContinuationToken = response.NextContinuationToken;
} while (!string.IsNullOrEmpty(listRequest.ContinuationToken));

Console.WriteLine($"Migration complete. Migrated {migrated} objects. Skipped {skipped} directory markers.");
return 0;

static AmazonS3Client CreateS3Client(MigrationOptions options, AmazonS3Config config)
{
    if (!string.IsNullOrWhiteSpace(options.S3AccessKey) && !string.IsNullOrWhiteSpace(options.S3SecretKey))
    {
        var credentials = new BasicAWSCredentials(options.S3AccessKey, options.S3SecretKey);
        return new AmazonS3Client(credentials, config);
    }

    return new AmazonS3Client(config);
}

static HttpClient CreateRustFsClient(MigrationOptions options)
{
    var client = new HttpClient
    {
        BaseAddress = new Uri(options.RustFsEndpoint, UriKind.Absolute),
        Timeout = TimeSpan.FromMinutes(5),
    };

    if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
    {
        client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFsApiKeyHeader, options.RustFsApiKey);
    }

    return client;
}

static Uri BuildRustFsUri(MigrationOptions options, string key)
{
    var normalized = string.Join('/', key
        .Split('/', StringSplitOptions.RemoveEmptyEntries)
        .Select(Uri.EscapeDataString));

    var builder = new UriBuilder(options.RustFsEndpoint)
    {
        Path = $"/api/v1/buckets/{Uri.EscapeDataString(options.RustFsBucket)}/objects/{normalized}",
    };

    return builder.Uri;
}

internal sealed record MigrationOptions
{
    public string S3Bucket { get; init; } = string.Empty;

    public string? S3ServiceUrl { get; init; }
        = null;

    public string? S3Region { get; init; }
        = null;

    public string? S3AccessKey { get; init; }
        = null;

    public string? S3SecretKey { get; init; }
        = null;

    public string RustFsEndpoint { get; init; } = string.Empty;

    public string RustFsBucket { get; init; } = string.Empty;

    public string? RustFsApiKeyHeader { get; init; }
        = null;

    public string? RustFsApiKey { get; init; }
        = null;

    public string? Prefix { get; init; }
        = null;

    public bool Immutable { get; init; }
        = false;

    public int? RetentionSeconds { get; init; }
        = null;

    public bool DryRun { get; init; }
        = false;

    public static MigrationOptions? Parse(string[] args)
    {
        var builder = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);

        for (var i = 0; i < args.Length; i++)
        {
            var key = args[i];
            if (key.StartsWith("--", StringComparison.OrdinalIgnoreCase))
            {
                var normalized = key[2..];
                if (string.Equals(normalized, "immutable", StringComparison.OrdinalIgnoreCase) || string.Equals(normalized, "dry-run", StringComparison.OrdinalIgnoreCase))
                {
                    builder[normalized] = "true";
                    continue;
                }

                if (i + 1 >= args.Length)
                {
                    Console.Error.WriteLine($"Missing value for argument '{key}'.");
                    return null;
                }

                builder[normalized] = args[++i];
            }
        }

        if (!builder.TryGetValue("s3-bucket", out var bucket) || string.IsNullOrWhiteSpace(bucket))
        {
            Console.Error.WriteLine("--s3-bucket is required.");
            return null;
        }

        if (!builder.TryGetValue("rustfs-endpoint", out var rustFsEndpoint) || string.IsNullOrWhiteSpace(rustFsEndpoint))
        {
            Console.Error.WriteLine("--rustfs-endpoint is required.");
            return null;
        }

        if (!builder.TryGetValue("rustfs-bucket", out var rustFsBucket) || string.IsNullOrWhiteSpace(rustFsBucket))
        {
            Console.Error.WriteLine("--rustfs-bucket is required.");
            return null;
        }

        int? retentionSeconds = null;
        if (builder.TryGetValue("retain-days", out var retainStr) && !string.IsNullOrWhiteSpace(retainStr))
        {
            if (double.TryParse(retainStr, out var days) && days > 0)
            {
                retentionSeconds = (int)Math.Ceiling(days * 24 * 60 * 60);
            }
            else
            {
                Console.Error.WriteLine("--retain-days must be a positive number.");
                return null;
            }
        }

        return new MigrationOptions
        {
            S3Bucket = bucket,
            S3ServiceUrl = builder.TryGetValue("s3-endpoint", out var s3Endpoint) ? s3Endpoint : null,
            S3Region = builder.TryGetValue("s3-region", out var s3Region) ? s3Region : null,
            S3AccessKey = builder.TryGetValue("s3-access-key", out var s3AccessKey) ? s3AccessKey : null,
            S3SecretKey = builder.TryGetValue("s3-secret-key", out var s3SecretKey) ? s3SecretKey : null,
            RustFsEndpoint = rustFsEndpoint!,
            RustFsBucket = rustFsBucket!,
            RustFsApiKeyHeader = builder.TryGetValue("rustfs-api-key-header", out var apiKeyHeader) ? apiKeyHeader : null,
            RustFsApiKey = builder.TryGetValue("rustfs-api-key", out var apiKey) ? apiKey : null,
            Prefix = builder.TryGetValue("prefix", out var prefix) ? prefix : null,
            Immutable = builder.ContainsKey("immutable"),
            RetentionSeconds = retentionSeconds,
            DryRun = builder.ContainsKey("dry-run"),
        };
    }

    public static void PrintUsage()
    {
        Console.WriteLine(@"Usage: dotnet run --project tools/RustFsMigrator -- \
    --s3-bucket <name> \
    [--s3-endpoint http://minio:9000] \
    [--s3-region us-east-1] \
    [--s3-access-key key --s3-secret-key secret] \
    --rustfs-endpoint http://rustfs:8080 \
    --rustfs-bucket scanner-artifacts \
    [--rustfs-api-key-header X-API-Key --rustfs-api-key token] \
    [--prefix scanner/] \
    [--immutable] \
    [--retain-days 365] \
    [--dry-run]");
    }
}