feat: Add RustFS artifact object store and migration tool
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS.
- Added unit tests for RustFsArtifactObjectStore functionality.
- Created a RustFS migrator tool to transfer objects from S3 to RustFS.
- Introduced policy preview and report models for API integration.
- Added fixtures and tests for policy preview and report functionality.
- Included necessary metadata and scripts for cache_pkg package.
This commit is contained in:
Vladimir Moushkov
2025-10-23 18:53:18 +03:00
parent aaa5fbfb78
commit f4d7a15a00
117 changed files with 4849 additions and 725 deletions

View File

@@ -0,0 +1,168 @@
using System.Linq;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Scanner.Storage;
using StellaOps.Scanner.Storage.ObjectStore;
using Xunit;
namespace StellaOps.Scanner.Storage.Tests;
public sealed class RustFsArtifactObjectStoreTests
{
[Fact]
public async Task PutAsync_PreservesStreamAndSendsImmutableHeaders()
{
var handler = new RecordingHttpMessageHandler();
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.OK));
var factory = new SingleHttpClientFactory(new HttpClient(handler)
{
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
});
var options = Options.Create(new ScannerStorageOptions
{
ObjectStore =
{
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
BucketName = "scanner-artifacts",
RustFs =
{
BaseUrl = "https://rustfs.test/api/v1/",
Timeout = TimeSpan.FromSeconds(10),
},
},
});
options.Value.ObjectStore.Headers["X-Custom-Header"] = "custom-value";
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
var payload = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("rustfs artifact payload"));
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/layers/digest/file.bin", true, TimeSpan.FromHours(1));
await store.PutAsync(descriptor, payload, CancellationToken.None);
Assert.True(payload.CanRead);
Assert.Equal(0, payload.Position);
var request = Assert.Single(handler.CapturedRequests);
Assert.Equal(HttpMethod.Put, request.Method);
Assert.Equal("https://rustfs.test/api/v1/buckets/scanner-artifacts/objects/scanner/layers/digest/file.bin", request.RequestUri.ToString());
Assert.Contains("X-Custom-Header", request.Headers.Keys);
Assert.Equal("custom-value", Assert.Single(request.Headers["X-Custom-Header"]));
Assert.Equal("true", Assert.Single(request.Headers["X-RustFS-Immutable"]));
Assert.Equal("3600", Assert.Single(request.Headers["X-RustFS-Retain-Seconds"]));
Assert.Equal("application/octet-stream", Assert.Single(request.Headers["Content-Type"]));
}
[Fact]
public async Task GetAsync_ReturnsNullOnNotFound()
{
var handler = new RecordingHttpMessageHandler();
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.NotFound));
var factory = new SingleHttpClientFactory(new HttpClient(handler)
{
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
});
var options = Options.Create(new ScannerStorageOptions
{
ObjectStore =
{
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
BucketName = "scanner-artifacts",
RustFs =
{
BaseUrl = "https://rustfs.test/api/v1/",
},
},
});
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/indexes/digest/index.bin", false);
var result = await store.GetAsync(descriptor, CancellationToken.None);
Assert.Null(result);
var request = Assert.Single(handler.CapturedRequests);
Assert.Equal(HttpMethod.Get, request.Method);
}
[Fact]
public async Task DeleteAsync_IgnoresNotFound()
{
var handler = new RecordingHttpMessageHandler();
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.NotFound));
var factory = new SingleHttpClientFactory(new HttpClient(handler)
{
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
});
var options = Options.Create(new ScannerStorageOptions
{
ObjectStore =
{
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
BucketName = "scanner-artifacts",
RustFs =
{
BaseUrl = "https://rustfs.test/api/v1/",
},
},
});
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/attest/digest/attest.bin", false);
await store.DeleteAsync(descriptor, CancellationToken.None);
var request = Assert.Single(handler.CapturedRequests);
Assert.Equal(HttpMethod.Delete, request.Method);
}
private sealed record CapturedRequest(HttpMethod Method, Uri RequestUri, IReadOnlyDictionary<string, string[]> Headers);
private sealed class RecordingHttpMessageHandler : HttpMessageHandler
{
public Queue<HttpResponseMessage> Responses { get; } = new();
public List<CapturedRequest> CapturedRequests { get; } = new();
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var headerSnapshot = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
foreach (var header in request.Headers)
{
headerSnapshot[header.Key] = header.Value.ToArray();
}
if (request.Content is not null)
{
foreach (var header in request.Content.Headers)
{
headerSnapshot[header.Key] = header.Value.ToArray();
}
// Materialize content to ensure downstream callers can inspect it.
_ = await request.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
}
CapturedRequests.Add(new CapturedRequest(request.Method, request.RequestUri!, headerSnapshot));
return Responses.Count > 0 ? Responses.Dequeue() : new HttpResponseMessage(HttpStatusCode.OK);
}
}
private sealed class SingleHttpClientFactory : IHttpClientFactory
{
private readonly HttpClient _client;
public SingleHttpClientFactory(HttpClient client)
{
_client = client;
}
public HttpClient CreateClient(string name) => _client;
}
}