Restructure solution layout by module
This commit is contained in:
master
@@ -0,0 +1,34 @@
|
||||
using System.Collections.Concurrent;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Tests;
|
||||
|
||||
internal sealed class InMemoryArtifactObjectStore : IArtifactObjectStore
|
||||
{
|
||||
private readonly ConcurrentDictionary<(string Bucket, string Key), byte[]> _objects = new();
|
||||
|
||||
public IReadOnlyDictionary<(string Bucket, string Key), byte[]> Objects => _objects;
|
||||
|
||||
public Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
|
||||
{
|
||||
_objects.TryRemove((descriptor.Bucket, descriptor.Key), out _);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<Stream?> GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_objects.TryGetValue((descriptor.Bucket, descriptor.Key), out var bytes))
|
||||
{
|
||||
return Task.FromResult<Stream?>(new MemoryStream(bytes, writable: false));
|
||||
}
|
||||
|
||||
return Task.FromResult<Stream?>(null);
|
||||
}
|
||||
|
||||
public async Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken)
|
||||
{
|
||||
using var buffer = new MemoryStream();
|
||||
await content.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false);
|
||||
_objects[(descriptor.Bucket, descriptor.Key)] = buffer.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Tests;
|
||||
|
||||
public sealed class RustFsArtifactObjectStoreTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task PutAsync_PreservesStreamAndSendsImmutableHeaders()
|
||||
{
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.OK));
|
||||
var factory = new SingleHttpClientFactory(new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
|
||||
});
|
||||
|
||||
var options = Options.Create(new ScannerStorageOptions
|
||||
{
|
||||
ObjectStore =
|
||||
{
|
||||
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
|
||||
BucketName = "scanner-artifacts",
|
||||
RustFs =
|
||||
{
|
||||
BaseUrl = "https://rustfs.test/api/v1/",
|
||||
Timeout = TimeSpan.FromSeconds(10),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
options.Value.ObjectStore.Headers["X-Custom-Header"] = "custom-value";
|
||||
|
||||
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
|
||||
|
||||
var payload = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("rustfs artifact payload"));
|
||||
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/layers/digest/file.bin", true, TimeSpan.FromHours(1));
|
||||
|
||||
await store.PutAsync(descriptor, payload, CancellationToken.None);
|
||||
|
||||
Assert.True(payload.CanRead);
|
||||
Assert.Equal(0, payload.Position);
|
||||
|
||||
var request = Assert.Single(handler.CapturedRequests);
|
||||
Assert.Equal(HttpMethod.Put, request.Method);
|
||||
Assert.Equal("https://rustfs.test/api/v1/buckets/scanner-artifacts/objects/scanner/layers/digest/file.bin", request.RequestUri.ToString());
|
||||
Assert.Contains("X-Custom-Header", request.Headers.Keys);
|
||||
Assert.Equal("custom-value", Assert.Single(request.Headers["X-Custom-Header"]));
|
||||
Assert.Equal("true", Assert.Single(request.Headers["X-RustFS-Immutable"]));
|
||||
Assert.Equal("3600", Assert.Single(request.Headers["X-RustFS-Retain-Seconds"]));
|
||||
Assert.Equal("application/octet-stream", Assert.Single(request.Headers["Content-Type"]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAsync_ReturnsNullOnNotFound()
|
||||
{
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.NotFound));
|
||||
var factory = new SingleHttpClientFactory(new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
|
||||
});
|
||||
|
||||
var options = Options.Create(new ScannerStorageOptions
|
||||
{
|
||||
ObjectStore =
|
||||
{
|
||||
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
|
||||
BucketName = "scanner-artifacts",
|
||||
RustFs =
|
||||
{
|
||||
BaseUrl = "https://rustfs.test/api/v1/",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
|
||||
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/indexes/digest/index.bin", false);
|
||||
|
||||
var result = await store.GetAsync(descriptor, CancellationToken.None);
|
||||
|
||||
Assert.Null(result);
|
||||
var request = Assert.Single(handler.CapturedRequests);
|
||||
Assert.Equal(HttpMethod.Get, request.Method);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeleteAsync_IgnoresNotFound()
|
||||
{
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.NotFound));
|
||||
var factory = new SingleHttpClientFactory(new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri("https://rustfs.test/api/v1/"),
|
||||
});
|
||||
|
||||
var options = Options.Create(new ScannerStorageOptions
|
||||
{
|
||||
ObjectStore =
|
||||
{
|
||||
Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs,
|
||||
BucketName = "scanner-artifacts",
|
||||
RustFs =
|
||||
{
|
||||
BaseUrl = "https://rustfs.test/api/v1/",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
var store = new RustFsArtifactObjectStore(factory, options, NullLogger<RustFsArtifactObjectStore>.Instance);
|
||||
var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/attest/digest/attest.bin", false);
|
||||
|
||||
await store.DeleteAsync(descriptor, CancellationToken.None);
|
||||
|
||||
var request = Assert.Single(handler.CapturedRequests);
|
||||
Assert.Equal(HttpMethod.Delete, request.Method);
|
||||
}
|
||||
|
||||
private sealed record CapturedRequest(HttpMethod Method, Uri RequestUri, IReadOnlyDictionary<string, string[]> Headers);
|
||||
|
||||
private sealed class RecordingHttpMessageHandler : HttpMessageHandler
|
||||
{
|
||||
public Queue<HttpResponseMessage> Responses { get; } = new();
|
||||
|
||||
public List<CapturedRequest> CapturedRequests { get; } = new();
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
var headerSnapshot = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var header in request.Headers)
|
||||
{
|
||||
headerSnapshot[header.Key] = header.Value.ToArray();
|
||||
}
|
||||
|
||||
if (request.Content is not null)
|
||||
{
|
||||
foreach (var header in request.Content.Headers)
|
||||
{
|
||||
headerSnapshot[header.Key] = header.Value.ToArray();
|
||||
}
|
||||
|
||||
// Materialize content to ensure downstream callers can inspect it.
|
||||
_ = await request.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
CapturedRequests.Add(new CapturedRequest(request.Method, request.RequestUri!, headerSnapshot));
|
||||
return Responses.Count > 0 ? Responses.Dequeue() : new HttpResponseMessage(HttpStatusCode.OK);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class SingleHttpClientFactory : IHttpClientFactory
|
||||
{
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public SingleHttpClientFactory(HttpClient client)
|
||||
{
|
||||
_client = client;
|
||||
}
|
||||
|
||||
public HttpClient CreateClient(string name) => _client;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using Mongo2Go;
|
||||
using MongoDB.Driver;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Tests;
|
||||
|
||||
public sealed class ScannerMongoFixture : IAsyncLifetime
|
||||
{
|
||||
public MongoDbRunner Runner { get; private set; } = null!;
|
||||
public IMongoClient Client { get; private set; } = null!;
|
||||
public IMongoDatabase Database { get; private set; } = null!;
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
Runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
Client = new MongoClient(Runner.ConnectionString);
|
||||
Database = Client.GetDatabase($"scanner-tests-{Guid.NewGuid():N}");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
Runner.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,149 @@
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.Catalog;
|
||||
using StellaOps.Scanner.Storage.Migrations;
|
||||
using StellaOps.Scanner.Storage.Mongo;
|
||||
using StellaOps.Scanner.Storage.ObjectStore;
|
||||
using StellaOps.Scanner.Storage.Repositories;
|
||||
using StellaOps.Scanner.Storage.Services;
|
||||
using Xunit;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Tests;
|
||||
|
||||
[CollectionDefinition("scanner-mongo-fixture")]
|
||||
public sealed class ScannerMongoCollection : ICollectionFixture<ScannerMongoFixture>
|
||||
{
|
||||
}
|
||||
|
||||
[Collection("scanner-mongo-fixture")]
|
||||
public sealed class StorageDualWriteFixture
|
||||
{
|
||||
private readonly ScannerMongoFixture _fixture;
|
||||
|
||||
public StorageDualWriteFixture(ScannerMongoFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StoreArtifactAsync_DualWrite_WritesToMirrorAndCatalog()
|
||||
{
|
||||
var options = BuildOptions(dualWrite: true, mirrorBucket: "mirror-bucket");
|
||||
var objectStore = new InMemoryArtifactObjectStore();
|
||||
var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero));
|
||||
|
||||
await InitializeMongoAsync(options);
|
||||
var provider = new MongoCollectionProvider(_fixture.Database, Options.Create(options));
|
||||
var artifactRepository = new ArtifactRepository(provider, fakeTime);
|
||||
var lifecycleRepository = new LifecycleRuleRepository(provider, fakeTime);
|
||||
var service = new ArtifactStorageService(
|
||||
artifactRepository,
|
||||
lifecycleRepository,
|
||||
objectStore,
|
||||
Options.Create(options),
|
||||
NullLogger<ArtifactStorageService>.Instance,
|
||||
fakeTime);
|
||||
|
||||
var bytes = System.Text.Encoding.UTF8.GetBytes("test artifact payload");
|
||||
using var stream = new MemoryStream(bytes);
|
||||
var expiresAt = DateTime.UtcNow.AddHours(6);
|
||||
var expectedTimestamp = fakeTime.GetUtcNow().UtcDateTime;
|
||||
|
||||
var document = await service.StoreArtifactAsync(
|
||||
ArtifactDocumentType.LayerBom,
|
||||
ArtifactDocumentFormat.CycloneDxJson,
|
||||
mediaType: "application/vnd.cyclonedx+json",
|
||||
content: stream,
|
||||
immutable: true,
|
||||
ttlClass: "compliance",
|
||||
expiresAtUtc: expiresAt,
|
||||
cancellationToken: CancellationToken.None);
|
||||
|
||||
var digest = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
|
||||
var expectedKey = $"{options.ObjectStore.RootPrefix.TrimEnd('/')}/layers/{digest}/sbom.cdx.json";
|
||||
Assert.Contains(objectStore.Objects.Keys, key => key.Bucket == options.ObjectStore.BucketName && key.Key == expectedKey);
|
||||
Assert.Contains(objectStore.Objects.Keys, key => key.Bucket == options.DualWrite.MirrorBucket && key.Key == expectedKey);
|
||||
|
||||
var artifact = await artifactRepository.GetAsync(document.Id, CancellationToken.None);
|
||||
Assert.NotNull(artifact);
|
||||
Assert.Equal($"sha256:{digest}", artifact!.BytesSha256);
|
||||
Assert.Equal(1, artifact.RefCount);
|
||||
Assert.Equal("compliance", artifact.TtlClass);
|
||||
Assert.True(artifact.Immutable);
|
||||
Assert.Equal(expectedTimestamp, artifact.CreatedAtUtc);
|
||||
Assert.Equal(expectedTimestamp, artifact.UpdatedAtUtc);
|
||||
|
||||
var lifecycleCollection = _fixture.Database.GetCollection<LifecycleRuleDocument>(ScannerStorageDefaults.Collections.LifecycleRules);
|
||||
var lifecycle = await lifecycleCollection.Find(x => x.ArtifactId == document.Id).FirstOrDefaultAsync();
|
||||
Assert.NotNull(lifecycle);
|
||||
Assert.Equal("compliance", lifecycle!.Class);
|
||||
Assert.True(lifecycle.ExpiresAtUtc.HasValue);
|
||||
Assert.True(lifecycle.ExpiresAtUtc.Value <= expiresAt.AddSeconds(5));
|
||||
Assert.Equal(expectedTimestamp, lifecycle.CreatedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Bootstrapper_CreatesLifecycleTtlIndex()
|
||||
{
|
||||
var options = BuildOptions(dualWrite: false, mirrorBucket: null);
|
||||
await InitializeMongoAsync(options);
|
||||
|
||||
var collection = _fixture.Database.GetCollection<BsonDocument>(ScannerStorageDefaults.Collections.LifecycleRules);
|
||||
var cursor = await collection.Indexes.ListAsync();
|
||||
var indexes = await cursor.ToListAsync();
|
||||
var ttlIndex = indexes.SingleOrDefault(x => string.Equals(x["name"].AsString, "lifecycle_expiresAt", StringComparison.Ordinal));
|
||||
|
||||
Assert.NotNull(ttlIndex);
|
||||
Assert.True(ttlIndex!.TryGetValue("expireAfterSeconds", out var expireValue));
|
||||
Assert.Equal(0, expireValue.ToInt64());
|
||||
|
||||
var uniqueIndex = indexes.SingleOrDefault(x => string.Equals(x["name"].AsString, "lifecycle_artifact_class", StringComparison.Ordinal));
|
||||
Assert.NotNull(uniqueIndex);
|
||||
Assert.True(uniqueIndex!["unique"].AsBoolean);
|
||||
}
|
||||
|
||||
private ScannerStorageOptions BuildOptions(bool dualWrite, string? mirrorBucket)
|
||||
{
|
||||
var options = new ScannerStorageOptions
|
||||
{
|
||||
Mongo =
|
||||
{
|
||||
ConnectionString = _fixture.Runner.ConnectionString,
|
||||
DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName,
|
||||
},
|
||||
ObjectStore =
|
||||
{
|
||||
BucketName = "primary-bucket",
|
||||
RootPrefix = "scanner",
|
||||
EnableObjectLock = true,
|
||||
},
|
||||
};
|
||||
|
||||
options.DualWrite.Enabled = dualWrite;
|
||||
options.DualWrite.MirrorBucket = mirrorBucket;
|
||||
return options;
|
||||
}
|
||||
|
||||
private async Task InitializeMongoAsync(ScannerStorageOptions options)
|
||||
{
|
||||
await _fixture.Client.DropDatabaseAsync(options.Mongo.DatabaseName);
|
||||
var migrations = new IMongoMigration[] { new EnsureLifecycleRuleTtlMigration() };
|
||||
var runner = new MongoMigrationRunner(
|
||||
_fixture.Database,
|
||||
migrations,
|
||||
NullLogger<MongoMigrationRunner>.Instance,
|
||||
TimeProvider.System);
|
||||
var bootstrapper = new MongoBootstrapper(
|
||||
_fixture.Database,
|
||||
Options.Create(options),
|
||||
NullLogger<MongoBootstrapper>.Instance,
|
||||
runner);
|
||||
|
||||
await bootstrapper.InitializeAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user