audit remarks work

2025-12-30 16:10:34 +02:00
parent e6ee092c7a
commit c706b3d3e0
72 changed files with 9997 additions and 5323 deletions
--- a/src/Tools/RustFsMigrator/Program.cs
+++ b/src/Tools/RustFsMigrator/Program.cs
@@ -1,8 +1,10 @@
+using System.Globalization;
+using System.Net;
+using System.Net.Http.Headers;
 using Amazon;
 using Amazon.Runtime;
 using Amazon.S3;
 using Amazon.S3.Model;
-using System.Net.Http.Headers;

 var options = MigrationOptions.Parse(args);
 if (options is null)
@@ -36,6 +38,11 @@ if (!string.IsNullOrWhiteSpace(options.S3Region))
 using var s3Client = CreateS3Client(options, s3Config);
 using var httpClient = CreateRustFsClient(options);

+using var cts = options.TimeoutSeconds > 0
+    ? new CancellationTokenSource(TimeSpan.FromSeconds(options.TimeoutSeconds))
+    : null;
+var cancellationToken = cts?.Token ?? CancellationToken.None;
+
 var listRequest = new ListObjectsV2Request
 {
    BucketName = options.S3Bucket,
@@ -46,69 +53,52 @@ var listRequest = new ListObjectsV2Request
 var migrated = 0;
 var skipped = 0;

-do
+try
 {
-    var response = await s3Client.ListObjectsV2Async(listRequest).ConfigureAwait(false);
-    foreach (var entry in response.S3Objects)
+    do
    {
-        if (entry.Size == 0 && entry.Key.EndsWith('/'))
+        var response = await ExecuteWithRetriesAsync<ListObjectsV2Response>(
+            token => s3Client.ListObjectsV2Async(listRequest, token),
+            "ListObjectsV2",
+            options,
+            cancellationToken).ConfigureAwait(false);
+
+        foreach (var entry in response.S3Objects)
        {
-            skipped++;
-            continue;
+            if (entry.Size == 0 && entry.Key.EndsWith("/", StringComparison.Ordinal))
+            {
+                skipped++;
+                continue;
+            }
+
+            Console.WriteLine($"Migrating {entry.Key} ({entry.Size} bytes)...");
+
+            if (options.DryRun)
+            {
+                migrated++;
+                continue;
+            }
+
+            try
+            {
+                await UploadObjectAsync(s3Client, httpClient, options, entry, cancellationToken).ConfigureAwait(false);
+                migrated++;
+            }
+            catch (Exception ex) when (ex is not OperationCanceledException)
+            {
+                Console.Error.WriteLine($"Failed to upload {entry.Key}: {ex.Message}");
+                return 2;
+            }
        }

-        Console.WriteLine($"Migrating {entry.Key} ({entry.Size} bytes)...");
-
-        if (options.DryRun)
-        {
-            migrated++;
-            continue;
-        }
-
-        using var getResponse = await s3Client.GetObjectAsync(new GetObjectRequest
-        {
-            BucketName = options.S3Bucket,
-            Key = entry.Key,
-        }).ConfigureAwait(false);
-
-        await using var memory = new MemoryStream();
-        await getResponse.ResponseStream.CopyToAsync(memory).ConfigureAwait(false);
-        memory.Position = 0;
-
-        using var request = new HttpRequestMessage(HttpMethod.Put, BuildRustFsUri(options, entry.Key))
-        {
-            Content = new ByteArrayContent(memory.ToArray()),
-        };
-        request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream");
-
-        if (options.Immutable)
-        {
-            request.Headers.TryAddWithoutValidation("X-RustFS-Immutable", "true");
-        }
-
-        if (options.RetentionSeconds is { } retainSeconds)
-        {
-            request.Headers.TryAddWithoutValidation("X-RustFS-Retain-Seconds", retainSeconds.ToString());
-        }
-
-        if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
-        {
-            request.Headers.TryAddWithoutValidation(options.RustFsApiKeyHeader!, options.RustFsApiKey!);
-        }
-
-        using var responseMessage = await httpClient.SendAsync(request).ConfigureAwait(false);
-        if (!responseMessage.IsSuccessStatusCode)
-        {
-            var error = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
-            Console.Error.WriteLine($"Failed to upload {entry.Key}: {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}\n{error}");
-            return 2;
-        }
-
-        migrated++;
-    }
-
-    listRequest.ContinuationToken = response.NextContinuationToken;
-} while (!string.IsNullOrEmpty(listRequest.ContinuationToken));
+        listRequest.ContinuationToken = response.NextContinuationToken;
+    } while (!string.IsNullOrEmpty(listRequest.ContinuationToken));
+}
+catch (OperationCanceledException)
+{
+    Console.Error.WriteLine("Migration canceled.");
+    return 3;
+}

 Console.WriteLine($"Migration complete. Migrated {migrated} objects. Skipped {skipped} directory markers.");
 return 0;
@@ -140,18 +130,112 @@ static HttpClient CreateRustFsClient(MigrationOptions options)
    return client;
 }

-static Uri BuildRustFsUri(MigrationOptions options, string key)
+static async Task UploadObjectAsync(IAmazonS3 s3Client, HttpClient httpClient, MigrationOptions options, S3Object entry, CancellationToken cancellationToken)
 {
-    var normalized = string.Join('/', key
-        .Split('/', StringSplitOptions.RemoveEmptyEntries)
-        .Select(Uri.EscapeDataString));
-
-    var builder = new UriBuilder(options.RustFsEndpoint)
+    await ExecuteWithRetriesAsync<object>(async token =>
    {
-        Path = $"/api/v1/buckets/{Uri.EscapeDataString(options.RustFsBucket)}/objects/{normalized}",
+        using var getResponse = await s3Client.GetObjectAsync(new GetObjectRequest
+        {
+            BucketName = options.S3Bucket,
+            Key = entry.Key,
+        }, token).ConfigureAwait(false);
+
+        using var request = BuildRustFsRequest(options, entry.Key, getResponse);
+        using var responseMessage = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token).ConfigureAwait(false);
+        if (!responseMessage.IsSuccessStatusCode)
+        {
+            var error = await responseMessage.Content.ReadAsStringAsync(token).ConfigureAwait(false);
+            if (ShouldRetry(responseMessage.StatusCode))
+            {
+                throw new RetryableException($"RustFS upload returned {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}: {error}");
+            }
+
+            throw new InvalidOperationException($"RustFS upload returned {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}: {error}");
+        }
+
+        return null!;
+    }, $"Upload {entry.Key}", options, cancellationToken).ConfigureAwait(false);
+}
+
+static HttpRequestMessage BuildRustFsRequest(MigrationOptions options, string key, GetObjectResponse getResponse)
+{
+    var request = new HttpRequestMessage(HttpMethod.Put, RustFsMigratorPaths.BuildRustFsUri(options, key))
+    {
+        Content = new StreamContent(getResponse.ResponseStream),
    };

-    return builder.Uri;
+    request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream");
+    if (getResponse.Headers.ContentLength > 0)
+    {
+        request.Content.Headers.ContentLength = getResponse.Headers.ContentLength;
+    }
+
+    if (options.Immutable)
+    {
+        request.Headers.TryAddWithoutValidation("X-RustFS-Immutable", "true");
+    }
+
+    if (options.RetentionSeconds is { } retainSeconds)
+    {
+        request.Headers.TryAddWithoutValidation("X-RustFS-Retain-Seconds", retainSeconds.ToString(CultureInfo.InvariantCulture));
+    }
+
+    if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
+    {
+        request.Headers.TryAddWithoutValidation(options.RustFsApiKeyHeader!, options.RustFsApiKey!);
+    }
+
+    return request;
+}
+
+static async Task<T> ExecuteWithRetriesAsync<T>(Func<CancellationToken, Task<T>> action, string operation, MigrationOptions options, CancellationToken cancellationToken)
+{
+    Exception? last = null;
+
+    for (var attempt = 1; attempt <= options.RetryAttempts; attempt++)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+
+        try
+        {
+            return await action(cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex) when (ShouldRetryException(ex) && attempt < options.RetryAttempts)
+        {
+            last = ex;
+            Console.Error.WriteLine($"[WARN] {operation} attempt {attempt} failed: {ex.Message}");
+            await Task.Delay(ComputeBackoffDelay(attempt, options.RetryDelayMs), cancellationToken).ConfigureAwait(false);
+        }
+    }
+
+    if (last is not null)
+    {
+        throw last;
+    }
+
+    return await action(cancellationToken).ConfigureAwait(false);
+}
+
+static TimeSpan ComputeBackoffDelay(int attempt, int retryDelayMs)
+{
+    var multiplier = Math.Pow(2, Math.Max(0, attempt - 1));
+    var delayMs = Math.Min(retryDelayMs * multiplier, 5000);
+    return TimeSpan.FromMilliseconds(delayMs);
+}
+
+static bool ShouldRetryException(Exception ex)
+    => ex is RetryableException or HttpRequestException or AmazonS3Exception or IOException;
+
+static bool ShouldRetry(HttpStatusCode statusCode)
+    => statusCode == HttpStatusCode.RequestTimeout
+       || statusCode == (HttpStatusCode)429
+       || (int)statusCode >= 500;
+
+internal sealed class RetryableException : Exception
+{
+    public RetryableException(string message) : base(message)
+    {
+    }
 }

 internal sealed record MigrationOptions
@@ -192,6 +276,15 @@ internal sealed record MigrationOptions
    public bool DryRun { get; init; }
        = false;

+    public int RetryAttempts { get; init; }
+        = 3;
+
+    public int RetryDelayMs { get; init; }
+        = 250;
+
+    public int TimeoutSeconds { get; init; }
+        = 0;
+
    public static MigrationOptions? Parse(string[] args)
    {
        var builder = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
@@ -202,7 +295,8 @@ internal sealed record MigrationOptions
            if (key.StartsWith("--", StringComparison.OrdinalIgnoreCase))
            {
                var normalized = key[2..];
-                if (string.Equals(normalized, "immutable", StringComparison.OrdinalIgnoreCase) || string.Equals(normalized, "dry-run", StringComparison.OrdinalIgnoreCase))
+                if (string.Equals(normalized, "immutable", StringComparison.OrdinalIgnoreCase) ||
+                    string.Equals(normalized, "dry-run", StringComparison.OrdinalIgnoreCase))
                {
                    builder[normalized] = "true";
                    continue;
@@ -239,7 +333,7 @@ internal sealed record MigrationOptions
        int? retentionSeconds = null;
        if (builder.TryGetValue("retain-days", out var retainStr) && !string.IsNullOrWhiteSpace(retainStr))
        {
-            if (double.TryParse(retainStr, out var days) && days > 0)
+            if (double.TryParse(retainStr, NumberStyles.Float, CultureInfo.InvariantCulture, out var days) && days > 0)
            {
                retentionSeconds = (int)Math.Ceiling(days * 24 * 60 * 60);
            }
@@ -250,6 +344,10 @@ internal sealed record MigrationOptions
            }
        }

+        var retryAttempts = ParseIntOption(builder, "retry-attempts", 3, min: 1, max: 10);
+        var retryDelayMs = ParseIntOption(builder, "retry-delay-ms", 250, min: 50, max: 2000);
+        var timeoutSeconds = ParseIntOption(builder, "timeout-seconds", 0, min: 0, max: 3600);
+
        return new MigrationOptions
        {
            S3Bucket = bucket,
@@ -265,6 +363,9 @@ internal sealed record MigrationOptions
            Immutable = builder.ContainsKey("immutable"),
            RetentionSeconds = retentionSeconds,
            DryRun = builder.ContainsKey("dry-run"),
+            RetryAttempts = retryAttempts,
+            RetryDelayMs = retryDelayMs,
+            TimeoutSeconds = timeoutSeconds,
        };
    }

@@ -281,6 +382,29 @@ internal sealed record MigrationOptions
    [--prefix scanner/] \
    [--immutable] \
    [--retain-days 365] \
+    [--retry-attempts 3] \
+    [--retry-delay-ms 250] \
+    [--timeout-seconds 0] \
    [--dry-run]");
    }
+
+    private static int ParseIntOption(Dictionary<string, string?> values, string name, int fallback, int min, int max)
+    {
+        if (!values.TryGetValue(name, out var raw) || string.IsNullOrWhiteSpace(raw))
+        {
+            return fallback;
+        }
+
+        if (!int.TryParse(raw, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
+        {
+            return fallback;
+        }
+
+        if (parsed < min)
+        {
+            return min;
+        }
+
+        return parsed > max ? max : parsed;
+    }
 }