feat: Add RustFS artifact object store and migration tool
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS. - Added unit tests for RustFsArtifactObjectStore functionality. - Created a RustFS migrator tool to transfer objects from S3 to RustFS. - Introduced policy preview and report models for API integration. - Added fixtures and tests for policy preview and report functionality. - Included necessary metadata and scripts for cache_pkg package.
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Tests;
|
||||
|
||||
public sealed class RustFsArtifactObjectStoreTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task PutAsync_PreservesStreamAndSendsImmutableHeaders()
|
||||
{
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.OK));
|
||||
var factory = new SingleHttpClientFactory(new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
|
||||
});
|
||||
|
||||
var options = Options.Create(new ScannerStorageOptions
|
||||
{
|
||||
ObjectStore =
|
||||
{
|
||||
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
|
||||
BucketName = "scanner-artifacts",
|
||||
RustFs =
|
||||
{
|
||||
BaseUrl = "https://rustfs.test/api/v1/",
|
||||
Timeout = TimeSpan.FromSeconds(10),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
options.Value.ObjectStore.Headers["X-Custom-Header"] = "custom-value";
|
||||
|
||||
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
|
||||
|
||||
var payload = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("rustfs artifact payload"));
|
||||
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/layers/digest/file.bin", true, TimeSpan.FromHours(1));
|
||||
|
||||
await store.PutAsync(descriptor, payload, CancellationToken.None);
|
||||
|
||||
Assert.True(payload.CanRead);
|
||||
Assert.Equal(0, payload.Position);
|
||||
|
||||
var request = Assert.Single(handler.CapturedRequests);
|
||||
Assert.Equal(HttpMethod.Put, request.Method);
|
||||
Assert.Equal("https://rustfs.test/api/v1/buckets/scanner-artifacts/objects/scanner/layers/digest/file.bin", request.RequestUri.ToString());
|
||||
Assert.Contains("X-Custom-Header", request.Headers.Keys);
|
||||
Assert.Equal("custom-value", Assert.Single(request.Headers["X-Custom-Header"]));
|
||||
Assert.Equal("true", Assert.Single(request.Headers["X-RustFS-Immutable"]));
|
||||
Assert.Equal("3600", Assert.Single(request.Headers["X-RustFS-Retain-Seconds"]));
|
||||
Assert.Equal("application/octet-stream", Assert.Single(request.Headers["Content-Type"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAsync_ReturnsNullOnNotFound()
|
||||
{
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.NotFound));
|
||||
var factory = new SingleHttpClientFactory(new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
|
||||
});
|
||||
|
||||
var options = Options.Create(new ScannerStorageOptions
|
||||
{
|
||||
ObjectStore =
|
||||
{
|
||||
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
|
||||
BucketName = "scanner-artifacts",
|
||||
RustFs =
|
||||
{
|
||||
BaseUrl = "https://rustfs.test/api/v1/",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
|
||||
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/indexes/digest/index.bin", false);
|
||||
|
||||
var result = await store.GetAsync(descriptor, CancellationToken.None);
|
||||
|
||||
Assert.Null(result);
|
||||
var request = Assert.Single(handler.CapturedRequests);
|
||||
Assert.Equal(HttpMethod.Get, request.Method);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteAsync_IgnoresNotFound()
|
||||
{
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.NotFound));
|
||||
var factory = new SingleHttpClientFactory(new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
|
||||
});
|
||||
|
||||
var options = Options.Create(new ScannerStorageOptions
|
||||
{
|
||||
ObjectStore =
|
||||
{
|
||||
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
|
||||
BucketName = "scanner-artifacts",
|
||||
RustFs =
|
||||
{
|
||||
BaseUrl = "https://rustfs.test/api/v1/",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
|
||||
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/attest/digest/attest.bin", false);
|
||||
|
||||
await store.DeleteAsync(descriptor, CancellationToken.None);
|
||||
|
||||
var request = Assert.Single(handler.CapturedRequests);
|
||||
Assert.Equal(HttpMethod.Delete, request.Method);
|
||||
}
|
||||
|
||||
private sealed record CapturedRequest(HttpMethod Method, Uri RequestUri, IReadOnlyDictionary<string, string[]> Headers);
|
||||
|
||||
private sealed class RecordingHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
public Queue<HttpResponseMessage> Responses { get; } = new();
|
||||
|
||||
public List<CapturedRequest> CapturedRequests { get; } = new();
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var headerSnapshot = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var header in request.Headers)
|
||||
{
|
||||
headerSnapshot[header.Key] = header.Value.ToArray();
|
||||
}
|
||||
|
||||
if (request.Content is not null)
|
||||
{
|
||||
foreach (var header in request.Content.Headers)
|
||||
{
|
||||
headerSnapshot[header.Key] = header.Value.ToArray();
|
||||
}
|
||||
|
||||
// Materialize content to ensure downstream callers can inspect it.
|
||||
_ = await request.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
CapturedRequests.Add(new CapturedRequest(request.Method, request.RequestUri!, headerSnapshot));
|
||||
return Responses.Count > 0 ? Responses.Dequeue() : new HttpResponseMessage(HttpStatusCode.OK);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SingleHttpClientFactory : IHttpClientFactory
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public SingleHttpClientFactory(HttpClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public HttpClient CreateClient(string name) => _client;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user