feat: Add RustFS artifact object store and migration tool

- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS.
- Added unit tests for RustFsArtifactObjectStore functionality.
- Created a RustFS migrator tool to transfer objects from S3 to RustFS.
- Introduced policy preview and report models for API integration.
- Added fixtures and tests for policy preview and report functionality.
- Included necessary metadata and scripts for cache_pkg package.

tools/RustFsMigrator/Program.cs

@@ -0,0 +1,286 @@
+using Amazon;
+using Amazon.Runtime;
+using Amazon.S3;
+using Amazon.S3.Model;
+using System.Net.Http.Headers;
+
+var options = MigrationOptions.Parse(args);
+if (options is null)
+{
+    MigrationOptions.PrintUsage();
+    return 1;
+}
+
+Console.WriteLine($"RustFS migrator starting (prefix: '{options.Prefix ?? "<all>"}')");
+if (options.DryRun)
+{
+    Console.WriteLine("Dry-run enabled. No objects will be written to RustFS.");
+}
+
+var s3Config = new AmazonS3Config
+{
+    ForcePathStyle = true,
+};
+
+if (!string.IsNullOrWhiteSpace(options.S3ServiceUrl))
+{
+    s3Config.ServiceURL = options.S3ServiceUrl;
+    s3Config.UseHttp = options.S3ServiceUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase);
+}
+
+if (!string.IsNullOrWhiteSpace(options.S3Region))
+{
+    s3Config.RegionEndpoint = RegionEndpoint.GetBySystemName(options.S3Region);
+}
+
+using var s3Client = CreateS3Client(options, s3Config);
+using var httpClient = CreateRustFsClient(options);
+
+var listRequest = new ListObjectsV2Request
+{
+    BucketName = options.S3Bucket,
+    Prefix = options.Prefix,
+    MaxKeys = 1000,
+};
+
+var migrated = 0;
+var skipped = 0;
+
+do
+{
+    var response = await s3Client.ListObjectsV2Async(listRequest).ConfigureAwait(false);
+    foreach (var entry in response.S3Objects)
+    {
+        if (entry.Size == 0 && entry.Key.EndsWith('/'))
+        {
+            skipped++;
+            continue;
+        }
+
+        Console.WriteLine($"Migrating {entry.Key} ({entry.Size} bytes)...");
+
+        if (options.DryRun)
+        {
+            migrated++;
+            continue;
+        }
+
+        using var getResponse = await s3Client.GetObjectAsync(new GetObjectRequest
+        {
+            BucketName = options.S3Bucket,
+            Key = entry.Key,
+        }).ConfigureAwait(false);
+
+        await using var memory = new MemoryStream();
+        await getResponse.ResponseStream.CopyToAsync(memory).ConfigureAwait(false);
+        memory.Position = 0;
+
+        using var request = new HttpRequestMessage(HttpMethod.Put, BuildRustFsUri(options, entry.Key))
+        {
+            Content = new ByteArrayContent(memory.ToArray()),
+        };
+        request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream");
+
+        if (options.Immutable)
+        {
+            request.Headers.TryAddWithoutValidation("X-RustFS-Immutable", "true");
+        }
+
+        if (options.RetentionSeconds is { } retainSeconds)
+        {
+            request.Headers.TryAddWithoutValidation("X-RustFS-Retain-Seconds", retainSeconds.ToString());
+        }
+
+        if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
+        {
+            request.Headers.TryAddWithoutValidation(options.RustFsApiKeyHeader!, options.RustFsApiKey!);
+        }
+
+        using var responseMessage = await httpClient.SendAsync(request).ConfigureAwait(false);
+        if (!responseMessage.IsSuccessStatusCode)
+        {
+            var error = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
+            Console.Error.WriteLine($"Failed to upload {entry.Key}: {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}\n{error}");
+            return 2;
+        }
+
+        migrated++;
+    }
+
+    listRequest.ContinuationToken = response.NextContinuationToken;
+} while (!string.IsNullOrEmpty(listRequest.ContinuationToken));
+
+Console.WriteLine($"Migration complete. Migrated {migrated} objects. Skipped {skipped} directory markers.");
+return 0;
+
+static AmazonS3Client CreateS3Client(MigrationOptions options, AmazonS3Config config)
+{
+    if (!string.IsNullOrWhiteSpace(options.S3AccessKey) && !string.IsNullOrWhiteSpace(options.S3SecretKey))
+    {
+        var credentials = new BasicAWSCredentials(options.S3AccessKey, options.S3SecretKey);
+        return new AmazonS3Client(credentials, config);
+    }
+
+    return new AmazonS3Client(config);
+}
+
+static HttpClient CreateRustFsClient(MigrationOptions options)
+{
+    var client = new HttpClient
+    {
+        BaseAddress = new Uri(options.RustFsEndpoint, UriKind.Absolute),
+        Timeout = TimeSpan.FromMinutes(5),
+    };
+
+    if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
+    {
+        client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFsApiKeyHeader, options.RustFsApiKey);
+    }
+
+    return client;
+}
+
+static Uri BuildRustFsUri(MigrationOptions options, string key)
+{
+    var normalized = string.Join('/', key
+        .Split('/', StringSplitOptions.RemoveEmptyEntries)
+        .Select(Uri.EscapeDataString));
+
+    var builder = new UriBuilder(options.RustFsEndpoint)
+    {
+        Path = $"/api/v1/buckets/{Uri.EscapeDataString(options.RustFsBucket)}/objects/{normalized}",
+    };
+
+    return builder.Uri;
+}
+
+internal sealed record MigrationOptions
+{
+    public string S3Bucket { get; init; } = string.Empty;
+
+    public string? S3ServiceUrl { get; init; }
+        = null;
+
+    public string? S3Region { get; init; }
+        = null;
+
+    public string? S3AccessKey { get; init; }
+        = null;
+
+    public string? S3SecretKey { get; init; }
+        = null;
+
+    public string RustFsEndpoint { get; init; } = string.Empty;
+
+    public string RustFsBucket { get; init; } = string.Empty;
+
+    public string? RustFsApiKeyHeader { get; init; }
+        = null;
+
+    public string? RustFsApiKey { get; init; }
+        = null;
+
+    public string? Prefix { get; init; }
+        = null;
+
+    public bool Immutable { get; init; }
+        = false;
+
+    public int? RetentionSeconds { get; init; }
+        = null;
+
+    public bool DryRun { get; init; }
+        = false;
+
+    public static MigrationOptions? Parse(string[] args)
+    {
+        var builder = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
+
+        for (var i = 0; i < args.Length; i++)
+        {
+            var key = args[i];
+            if (key.StartsWith("--", StringComparison.OrdinalIgnoreCase))
+            {
+                var normalized = key[2..];
+                if (string.Equals(normalized, "immutable", StringComparison.OrdinalIgnoreCase) || string.Equals(normalized, "dry-run", StringComparison.OrdinalIgnoreCase))
+                {
+                    builder[normalized] = "true";
+                    continue;
+                }
+
+                if (i + 1 >= args.Length)
+                {
+                    Console.Error.WriteLine($"Missing value for argument '{key}'.");
+                    return null;
+                }
+
+                builder[normalized] = args[++i];
+            }
+        }
+
+        if (!builder.TryGetValue("s3-bucket", out var bucket) || string.IsNullOrWhiteSpace(bucket))
+        {
+            Console.Error.WriteLine("--s3-bucket is required.");
+            return null;
+        }
+
+        if (!builder.TryGetValue("rustfs-endpoint", out var rustFsEndpoint) || string.IsNullOrWhiteSpace(rustFsEndpoint))
+        {
+            Console.Error.WriteLine("--rustfs-endpoint is required.");
+            return null;
+        }
+
+        if (!builder.TryGetValue("rustfs-bucket", out var rustFsBucket) || string.IsNullOrWhiteSpace(rustFsBucket))
+        {
+            Console.Error.WriteLine("--rustfs-bucket is required.");
+            return null;
+        }
+
+        int? retentionSeconds = null;
+        if (builder.TryGetValue("retain-days", out var retainStr) && !string.IsNullOrWhiteSpace(retainStr))
+        {
+            if (double.TryParse(retainStr, out var days) && days > 0)
+            {
+                retentionSeconds = (int)Math.Ceiling(days * 24 * 60 * 60);
+            }
+            else
+            {
+                Console.Error.WriteLine("--retain-days must be a positive number.");
+                return null;
+            }
+        }
+
+        return new MigrationOptions
+        {
+            S3Bucket = bucket,
+            S3ServiceUrl = builder.TryGetValue("s3-endpoint", out var s3Endpoint) ? s3Endpoint : null,
+            S3Region = builder.TryGetValue("s3-region", out var s3Region) ? s3Region : null,
+            S3AccessKey = builder.TryGetValue("s3-access-key", out var s3AccessKey) ? s3AccessKey : null,
+            S3SecretKey = builder.TryGetValue("s3-secret-key", out var s3SecretKey) ? s3SecretKey : null,
+            RustFsEndpoint = rustFsEndpoint!,
+            RustFsBucket = rustFsBucket!,
+            RustFsApiKeyHeader = builder.TryGetValue("rustfs-api-key-header", out var apiKeyHeader) ? apiKeyHeader : null,
+            RustFsApiKey = builder.TryGetValue("rustfs-api-key", out var apiKey) ? apiKey : null,
+            Prefix = builder.TryGetValue("prefix", out var prefix) ? prefix : null,
+            Immutable = builder.ContainsKey("immutable"),
+            RetentionSeconds = retentionSeconds,
+            DryRun = builder.ContainsKey("dry-run"),
+        };
+    }
+
+    public static void PrintUsage()
+    {
+        Console.WriteLine(@"Usage: dotnet run --project tools/RustFsMigrator -- \
+    --s3-bucket <name> \
+    [--s3-endpoint http://minio:9000] \
+    [--s3-region us-east-1] \
+    [--s3-access-key key --s3-secret-key secret] \
+    --rustfs-endpoint http://rustfs:8080 \
+    --rustfs-bucket scanner-artifacts \
+    [--rustfs-api-key-header X-API-Key --rustfs-api-key token] \
+    [--prefix scanner/] \
+    [--immutable] \
+    [--retain-days 365] \
+    [--dry-run]");
+    }
+}

tools/RustFsMigrator/RustFsMigrator.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.7.305.6" />
+  </ItemGroup>
+</Project>