using Amazon; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; using System.Net.Http.Headers; var options = MigrationOptions.Parse(args); if (options is null) { MigrationOptions.PrintUsage(); return 1; } Console.WriteLine($"RustFS migrator starting (prefix: '{options.Prefix ?? ""}')"); if (options.DryRun) { Console.WriteLine("Dry-run enabled. No objects will be written to RustFS."); } var s3Config = new AmazonS3Config { ForcePathStyle = true, }; if (!string.IsNullOrWhiteSpace(options.S3ServiceUrl)) { s3Config.ServiceURL = options.S3ServiceUrl; s3Config.UseHttp = options.S3ServiceUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase); } if (!string.IsNullOrWhiteSpace(options.S3Region)) { s3Config.RegionEndpoint = RegionEndpoint.GetBySystemName(options.S3Region); } using var s3Client = CreateS3Client(options, s3Config); using var httpClient = CreateRustFsClient(options); var listRequest = new ListObjectsV2Request { BucketName = options.S3Bucket, Prefix = options.Prefix, MaxKeys = 1000, }; var migrated = 0; var skipped = 0; do { var response = await s3Client.ListObjectsV2Async(listRequest).ConfigureAwait(false); foreach (var entry in response.S3Objects) { if (entry.Size == 0 && entry.Key.EndsWith('/')) { skipped++; continue; } Console.WriteLine($"Migrating {entry.Key} ({entry.Size} bytes)..."); if (options.DryRun) { migrated++; continue; } using var getResponse = await s3Client.GetObjectAsync(new GetObjectRequest { BucketName = options.S3Bucket, Key = entry.Key, }).ConfigureAwait(false); await using var memory = new MemoryStream(); await getResponse.ResponseStream.CopyToAsync(memory).ConfigureAwait(false); memory.Position = 0; using var request = new HttpRequestMessage(HttpMethod.Put, BuildRustFsUri(options, entry.Key)) { Content = new ByteArrayContent(memory.ToArray()), }; request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream"); if (options.Immutable) { request.Headers.TryAddWithoutValidation("X-RustFS-Immutable", "true"); } if (options.RetentionSeconds is { } retainSeconds) { request.Headers.TryAddWithoutValidation("X-RustFS-Retain-Seconds", retainSeconds.ToString()); } if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey)) { request.Headers.TryAddWithoutValidation(options.RustFsApiKeyHeader!, options.RustFsApiKey!); } using var responseMessage = await httpClient.SendAsync(request).ConfigureAwait(false); if (!responseMessage.IsSuccessStatusCode) { var error = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false); Console.Error.WriteLine($"Failed to upload {entry.Key}: {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}\n{error}"); return 2; } migrated++; } listRequest.ContinuationToken = response.NextContinuationToken; } while (!string.IsNullOrEmpty(listRequest.ContinuationToken)); Console.WriteLine($"Migration complete. Migrated {migrated} objects. Skipped {skipped} directory markers."); return 0; static AmazonS3Client CreateS3Client(MigrationOptions options, AmazonS3Config config) { if (!string.IsNullOrWhiteSpace(options.S3AccessKey) && !string.IsNullOrWhiteSpace(options.S3SecretKey)) { var credentials = new BasicAWSCredentials(options.S3AccessKey, options.S3SecretKey); return new AmazonS3Client(credentials, config); } return new AmazonS3Client(config); } static HttpClient CreateRustFsClient(MigrationOptions options) { var client = new HttpClient { BaseAddress = new Uri(options.RustFsEndpoint, UriKind.Absolute), Timeout = TimeSpan.FromMinutes(5), }; if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey)) { client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFsApiKeyHeader, options.RustFsApiKey); } return client; } static Uri BuildRustFsUri(MigrationOptions options, string key) { var normalized = string.Join('/', key .Split('/', StringSplitOptions.RemoveEmptyEntries) .Select(Uri.EscapeDataString)); var builder = new UriBuilder(options.RustFsEndpoint) { Path = $"/api/v1/buckets/{Uri.EscapeDataString(options.RustFsBucket)}/objects/{normalized}", }; return builder.Uri; } internal sealed record MigrationOptions { public string S3Bucket { get; init; } = string.Empty; public string? S3ServiceUrl { get; init; } = null; public string? S3Region { get; init; } = null; public string? S3AccessKey { get; init; } = null; public string? S3SecretKey { get; init; } = null; public string RustFsEndpoint { get; init; } = string.Empty; public string RustFsBucket { get; init; } = string.Empty; public string? RustFsApiKeyHeader { get; init; } = null; public string? RustFsApiKey { get; init; } = null; public string? Prefix { get; init; } = null; public bool Immutable { get; init; } = false; public int? RetentionSeconds { get; init; } = null; public bool DryRun { get; init; } = false; public static MigrationOptions? Parse(string[] args) { var builder = new Dictionary(StringComparer.OrdinalIgnoreCase); for (var i = 0; i < args.Length; i++) { var key = args[i]; if (key.StartsWith("--", StringComparison.OrdinalIgnoreCase)) { var normalized = key[2..]; if (string.Equals(normalized, "immutable", StringComparison.OrdinalIgnoreCase) || string.Equals(normalized, "dry-run", StringComparison.OrdinalIgnoreCase)) { builder[normalized] = "true"; continue; } if (i + 1 >= args.Length) { Console.Error.WriteLine($"Missing value for argument '{key}'."); return null; } builder[normalized] = args[++i]; } } if (!builder.TryGetValue("s3-bucket", out var bucket) || string.IsNullOrWhiteSpace(bucket)) { Console.Error.WriteLine("--s3-bucket is required."); return null; } if (!builder.TryGetValue("rustfs-endpoint", out var rustFsEndpoint) || string.IsNullOrWhiteSpace(rustFsEndpoint)) { Console.Error.WriteLine("--rustfs-endpoint is required."); return null; } if (!builder.TryGetValue("rustfs-bucket", out var rustFsBucket) || string.IsNullOrWhiteSpace(rustFsBucket)) { Console.Error.WriteLine("--rustfs-bucket is required."); return null; } int? retentionSeconds = null; if (builder.TryGetValue("retain-days", out var retainStr) && !string.IsNullOrWhiteSpace(retainStr)) { if (double.TryParse(retainStr, out var days) && days > 0) { retentionSeconds = (int)Math.Ceiling(days * 24 * 60 * 60); } else { Console.Error.WriteLine("--retain-days must be a positive number."); return null; } } return new MigrationOptions { S3Bucket = bucket, S3ServiceUrl = builder.TryGetValue("s3-endpoint", out var s3Endpoint) ? s3Endpoint : null, S3Region = builder.TryGetValue("s3-region", out var s3Region) ? s3Region : null, S3AccessKey = builder.TryGetValue("s3-access-key", out var s3AccessKey) ? s3AccessKey : null, S3SecretKey = builder.TryGetValue("s3-secret-key", out var s3SecretKey) ? s3SecretKey : null, RustFsEndpoint = rustFsEndpoint!, RustFsBucket = rustFsBucket!, RustFsApiKeyHeader = builder.TryGetValue("rustfs-api-key-header", out var apiKeyHeader) ? apiKeyHeader : null, RustFsApiKey = builder.TryGetValue("rustfs-api-key", out var apiKey) ? apiKey : null, Prefix = builder.TryGetValue("prefix", out var prefix) ? prefix : null, Immutable = builder.ContainsKey("immutable"), RetentionSeconds = retentionSeconds, DryRun = builder.ContainsKey("dry-run"), }; } public static void PrintUsage() { Console.WriteLine(@"Usage: dotnet run --project tools/RustFsMigrator -- \ --s3-bucket \ [--s3-endpoint http://minio:9000] \ [--s3-region us-east-1] \ [--s3-access-key key --s3-secret-key secret] \ --rustfs-endpoint http://rustfs:8080 \ --rustfs-bucket scanner-artifacts \ [--rustfs-api-key-header X-API-Key --rustfs-api-key token] \ [--prefix scanner/] \ [--immutable] \ [--retain-days 365] \ [--dry-run]"); } }