feat: Add RustFS artifact object store and migration tool
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS. - Added unit tests for RustFsArtifactObjectStore functionality. - Created a RustFS migrator tool to transfer objects from S3 to RustFS. - Introduced policy preview and report models for API integration. - Added fixtures and tests for policy preview and report functionality. - Included necessary metadata and scripts for cache_pkg package.
This commit is contained in:
Vladimir Moushkov
286
tools/RustFsMigrator/Program.cs
Normal file
286
tools/RustFsMigrator/Program.cs
Normal file
@@ -0,0 +1,286 @@
|
||||
using Amazon;
|
||||
using Amazon.Runtime;
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using System.Net.Http.Headers;
|
||||
|
||||
var options = MigrationOptions.Parse(args);
|
||||
if (options is null)
|
||||
{
|
||||
MigrationOptions.PrintUsage();
|
||||
return 1;
|
||||
}
|
||||
|
||||
Console.WriteLine($"RustFS migrator starting (prefix: '{options.Prefix ?? "<all>"}')");
|
||||
if (options.DryRun)
|
||||
{
|
||||
Console.WriteLine("Dry-run enabled. No objects will be written to RustFS.");
|
||||
}
|
||||
|
||||
var s3Config = new AmazonS3Config
|
||||
{
|
||||
ForcePathStyle = true,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.S3ServiceUrl))
|
||||
{
|
||||
s3Config.ServiceURL = options.S3ServiceUrl;
|
||||
s3Config.UseHttp = options.S3ServiceUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.S3Region))
|
||||
{
|
||||
s3Config.RegionEndpoint = RegionEndpoint.GetBySystemName(options.S3Region);
|
||||
}
|
||||
|
||||
using var s3Client = CreateS3Client(options, s3Config);
|
||||
using var httpClient = CreateRustFsClient(options);
|
||||
|
||||
var listRequest = new ListObjectsV2Request
|
||||
{
|
||||
BucketName = options.S3Bucket,
|
||||
Prefix = options.Prefix,
|
||||
MaxKeys = 1000,
|
||||
};
|
||||
|
||||
var migrated = 0;
|
||||
var skipped = 0;
|
||||
|
||||
do
|
||||
{
|
||||
var response = await s3Client.ListObjectsV2Async(listRequest).ConfigureAwait(false);
|
||||
foreach (var entry in response.S3Objects)
|
||||
{
|
||||
if (entry.Size == 0 && entry.Key.EndsWith('/'))
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Migrating {entry.Key} ({entry.Size} bytes)...");
|
||||
|
||||
if (options.DryRun)
|
||||
{
|
||||
migrated++;
|
||||
continue;
|
||||
}
|
||||
|
||||
using var getResponse = await s3Client.GetObjectAsync(new GetObjectRequest
|
||||
{
|
||||
BucketName = options.S3Bucket,
|
||||
Key = entry.Key,
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
await using var memory = new MemoryStream();
|
||||
await getResponse.ResponseStream.CopyToAsync(memory).ConfigureAwait(false);
|
||||
memory.Position = 0;
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Put, BuildRustFsUri(options, entry.Key))
|
||||
{
|
||||
Content = new ByteArrayContent(memory.ToArray()),
|
||||
};
|
||||
request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream");
|
||||
|
||||
if (options.Immutable)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation("X-RustFS-Immutable", "true");
|
||||
}
|
||||
|
||||
if (options.RetentionSeconds is { } retainSeconds)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation("X-RustFS-Retain-Seconds", retainSeconds.ToString());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(options.RustFsApiKeyHeader!, options.RustFsApiKey!);
|
||||
}
|
||||
|
||||
using var responseMessage = await httpClient.SendAsync(request).ConfigureAwait(false);
|
||||
if (!responseMessage.IsSuccessStatusCode)
|
||||
{
|
||||
var error = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
Console.Error.WriteLine($"Failed to upload {entry.Key}: {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}\n{error}");
|
||||
return 2;
|
||||
}
|
||||
|
||||
migrated++;
|
||||
}
|
||||
|
||||
listRequest.ContinuationToken = response.NextContinuationToken;
|
||||
} while (!string.IsNullOrEmpty(listRequest.ContinuationToken));
|
||||
|
||||
Console.WriteLine($"Migration complete. Migrated {migrated} objects. Skipped {skipped} directory markers.");
|
||||
return 0;
|
||||
|
||||
static AmazonS3Client CreateS3Client(MigrationOptions options, AmazonS3Config config)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.S3AccessKey) && !string.IsNullOrWhiteSpace(options.S3SecretKey))
|
||||
{
|
||||
var credentials = new BasicAWSCredentials(options.S3AccessKey, options.S3SecretKey);
|
||||
return new AmazonS3Client(credentials, config);
|
||||
}
|
||||
|
||||
return new AmazonS3Client(config);
|
||||
}
|
||||
|
||||
static HttpClient CreateRustFsClient(MigrationOptions options)
|
||||
{
|
||||
var client = new HttpClient
|
||||
{
|
||||
BaseAddress = new Uri(options.RustFsEndpoint, UriKind.Absolute),
|
||||
Timeout = TimeSpan.FromMinutes(5),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
|
||||
{
|
||||
client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFsApiKeyHeader, options.RustFsApiKey);
|
||||
}
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
static Uri BuildRustFsUri(MigrationOptions options, string key)
|
||||
{
|
||||
var normalized = string.Join('/', key
|
||||
.Split('/', StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(Uri.EscapeDataString));
|
||||
|
||||
var builder = new UriBuilder(options.RustFsEndpoint)
|
||||
{
|
||||
Path = $"/api/v1/buckets/{Uri.EscapeDataString(options.RustFsBucket)}/objects/{normalized}",
|
||||
};
|
||||
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
internal sealed record MigrationOptions
|
||||
{
|
||||
public string S3Bucket { get; init; } = string.Empty;
|
||||
|
||||
public string? S3ServiceUrl { get; init; }
|
||||
= null;
|
||||
|
||||
public string? S3Region { get; init; }
|
||||
= null;
|
||||
|
||||
public string? S3AccessKey { get; init; }
|
||||
= null;
|
||||
|
||||
public string? S3SecretKey { get; init; }
|
||||
= null;
|
||||
|
||||
public string RustFsEndpoint { get; init; } = string.Empty;
|
||||
|
||||
public string RustFsBucket { get; init; } = string.Empty;
|
||||
|
||||
public string? RustFsApiKeyHeader { get; init; }
|
||||
= null;
|
||||
|
||||
public string? RustFsApiKey { get; init; }
|
||||
= null;
|
||||
|
||||
public string? Prefix { get; init; }
|
||||
= null;
|
||||
|
||||
public bool Immutable { get; init; }
|
||||
= false;
|
||||
|
||||
public int? RetentionSeconds { get; init; }
|
||||
= null;
|
||||
|
||||
public bool DryRun { get; init; }
|
||||
= false;
|
||||
|
||||
public static MigrationOptions? Parse(string[] args)
|
||||
{
|
||||
var builder = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
var key = args[i];
|
||||
if (key.StartsWith("--", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var normalized = key[2..];
|
||||
if (string.Equals(normalized, "immutable", StringComparison.OrdinalIgnoreCase) || string.Equals(normalized, "dry-run", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
builder[normalized] = "true";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i + 1 >= args.Length)
|
||||
{
|
||||
Console.Error.WriteLine($"Missing value for argument '{key}'.");
|
||||
return null;
|
||||
}
|
||||
|
||||
builder[normalized] = args[++i];
|
||||
}
|
||||
}
|
||||
|
||||
if (!builder.TryGetValue("s3-bucket", out var bucket) || string.IsNullOrWhiteSpace(bucket))
|
||||
{
|
||||
Console.Error.WriteLine("--s3-bucket is required.");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!builder.TryGetValue("rustfs-endpoint", out var rustFsEndpoint) || string.IsNullOrWhiteSpace(rustFsEndpoint))
|
||||
{
|
||||
Console.Error.WriteLine("--rustfs-endpoint is required.");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!builder.TryGetValue("rustfs-bucket", out var rustFsBucket) || string.IsNullOrWhiteSpace(rustFsBucket))
|
||||
{
|
||||
Console.Error.WriteLine("--rustfs-bucket is required.");
|
||||
return null;
|
||||
}
|
||||
|
||||
int? retentionSeconds = null;
|
||||
if (builder.TryGetValue("retain-days", out var retainStr) && !string.IsNullOrWhiteSpace(retainStr))
|
||||
{
|
||||
if (double.TryParse(retainStr, out var days) && days > 0)
|
||||
{
|
||||
retentionSeconds = (int)Math.Ceiling(days * 24 * 60 * 60);
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.Error.WriteLine("--retain-days must be a positive number.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return new MigrationOptions
|
||||
{
|
||||
S3Bucket = bucket,
|
||||
S3ServiceUrl = builder.TryGetValue("s3-endpoint", out var s3Endpoint) ? s3Endpoint : null,
|
||||
S3Region = builder.TryGetValue("s3-region", out var s3Region) ? s3Region : null,
|
||||
S3AccessKey = builder.TryGetValue("s3-access-key", out var s3AccessKey) ? s3AccessKey : null,
|
||||
S3SecretKey = builder.TryGetValue("s3-secret-key", out var s3SecretKey) ? s3SecretKey : null,
|
||||
RustFsEndpoint = rustFsEndpoint!,
|
||||
RustFsBucket = rustFsBucket!,
|
||||
RustFsApiKeyHeader = builder.TryGetValue("rustfs-api-key-header", out var apiKeyHeader) ? apiKeyHeader : null,
|
||||
RustFsApiKey = builder.TryGetValue("rustfs-api-key", out var apiKey) ? apiKey : null,
|
||||
Prefix = builder.TryGetValue("prefix", out var prefix) ? prefix : null,
|
||||
Immutable = builder.ContainsKey("immutable"),
|
||||
RetentionSeconds = retentionSeconds,
|
||||
DryRun = builder.ContainsKey("dry-run"),
|
||||
};
|
||||
}
|
||||
|
||||
public static void PrintUsage()
|
||||
{
|
||||
Console.WriteLine(@"Usage: dotnet run --project tools/RustFsMigrator -- \
|
||||
--s3-bucket <name> \
|
||||
[--s3-endpoint http://minio:9000] \
|
||||
[--s3-region us-east-1] \
|
||||
[--s3-access-key key --s3-secret-key secret] \
|
||||
--rustfs-endpoint http://rustfs:8080 \
|
||||
--rustfs-bucket scanner-artifacts \
|
||||
[--rustfs-api-key-header X-API-Key --rustfs-api-key token] \
|
||||
[--prefix scanner/] \
|
||||
[--immutable] \
|
||||
[--retain-days 365] \
|
||||
[--dry-run]");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user