Resolve Concelier/Excititor merge conflicts

This commit is contained in:
master
2025-10-20 14:19:25 +03:00
2687 changed files with 212646 additions and 85913 deletions

View File

@@ -0,0 +1,277 @@
using System;
using System.Collections.Immutable;
using System.IO;
using System.Text;
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo;
using Xunit;
namespace StellaOps.Excititor.Export.Tests;
public sealed class ExportEngineTests
{
[Fact]
public async Task ExportAsync_GeneratesAndCachesManifest()
{
var store = new InMemoryExportStore();
var evaluator = new StaticPolicyEvaluator("baseline/v1");
var dataSource = new InMemoryExportDataSource();
var exporter = new DummyExporter(VexExportFormat.Json);
var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger<VexExportEngine>.Instance);
var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") });
var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow, ForceRefresh: false);
var manifest = await engine.ExportAsync(context, CancellationToken.None);
Assert.False(manifest.FromCache);
Assert.Equal(VexExportFormat.Json, manifest.Format);
Assert.Equal("baseline/v1", manifest.ConsensusRevision);
Assert.Equal(1, manifest.ClaimCount);
// second call hits cache
var cached = await engine.ExportAsync(context, CancellationToken.None);
Assert.True(cached.FromCache);
Assert.Equal(manifest.ExportId, cached.ExportId);
}
[Fact]
public async Task ExportAsync_ForceRefreshInvalidatesCacheEntry()
{
var store = new InMemoryExportStore();
var evaluator = new StaticPolicyEvaluator("baseline/v1");
var dataSource = new InMemoryExportDataSource();
var exporter = new DummyExporter(VexExportFormat.Json);
var cacheIndex = new RecordingCacheIndex();
var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger<VexExportEngine>.Instance, cacheIndex);
var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") });
var initialContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow);
_ = await engine.ExportAsync(initialContext, CancellationToken.None);
var refreshContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow.AddMinutes(1), ForceRefresh: true);
var refreshed = await engine.ExportAsync(refreshContext, CancellationToken.None);
Assert.False(refreshed.FromCache);
var signature = VexQuerySignature.FromQuery(refreshContext.Query);
Assert.True(cacheIndex.RemoveCalls.TryGetValue((signature.Value, refreshContext.Format), out var removed));
Assert.True(removed);
}
[Fact]
public async Task ExportAsync_WritesArtifactsToAllStores()
{
var store = new InMemoryExportStore();
var evaluator = new StaticPolicyEvaluator("baseline/v1");
var dataSource = new InMemoryExportDataSource();
var exporter = new DummyExporter(VexExportFormat.Json);
var recorder1 = new RecordingArtifactStore();
var recorder2 = new RecordingArtifactStore();
var engine = new VexExportEngine(
store,
evaluator,
dataSource,
new[] { exporter },
NullLogger<VexExportEngine>.Instance,
cacheIndex: null,
artifactStores: new[] { recorder1, recorder2 });
var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") });
var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow);
await engine.ExportAsync(context, CancellationToken.None);
Assert.Equal(1, recorder1.SaveCount);
Assert.Equal(1, recorder2.SaveCount);
}
[Fact]
public async Task ExportAsync_AttachesAttestationMetadata()
{
var store = new InMemoryExportStore();
var evaluator = new StaticPolicyEvaluator("baseline/v1");
var dataSource = new InMemoryExportDataSource();
var exporter = new DummyExporter(VexExportFormat.Json);
var attestation = new RecordingAttestationClient();
var engine = new VexExportEngine(
store,
evaluator,
dataSource,
new[] { exporter },
NullLogger<VexExportEngine>.Instance,
cacheIndex: null,
artifactStores: null,
attestationClient: attestation);
var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") });
var requestedAt = DateTimeOffset.UtcNow;
var context = new VexExportRequestContext(query, VexExportFormat.Json, requestedAt);
var manifest = await engine.ExportAsync(context, CancellationToken.None);
Assert.NotNull(attestation.LastRequest);
Assert.Equal(manifest.ExportId, attestation.LastRequest!.ExportId);
Assert.NotNull(manifest.Attestation);
Assert.Equal(attestation.Response.Attestation.EnvelopeDigest, manifest.Attestation!.EnvelopeDigest);
Assert.Equal(attestation.Response.Attestation.PredicateType, manifest.Attestation.PredicateType);
Assert.NotNull(store.LastSavedManifest);
Assert.Equal(manifest.Attestation, store.LastSavedManifest!.Attestation);
}
private sealed class InMemoryExportStore : IVexExportStore
{
private readonly Dictionary<string, VexExportManifest> _store = new(StringComparer.Ordinal);
public VexExportManifest? LastSavedManifest { get; private set; }
public ValueTask<VexExportManifest?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
var key = CreateKey(signature.Value, format);
_store.TryGetValue(key, out var manifest);
return ValueTask.FromResult<VexExportManifest?>(manifest);
}
public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null)
{
var key = CreateKey(manifest.QuerySignature.Value, manifest.Format);
_store[key] = manifest;
LastSavedManifest = manifest;
return ValueTask.CompletedTask;
}
private static string CreateKey(string signature, VexExportFormat format)
=> FormattableString.Invariant($"{signature}|{format}");
}
private sealed class RecordingAttestationClient : IVexAttestationClient
{
public VexAttestationRequest? LastRequest { get; private set; }
public VexAttestationResponse Response { get; } = new VexAttestationResponse(
new VexAttestationMetadata(
predicateType: "https://stella-ops.org/attestations/vex-export",
rekor: new VexRekorReference("0.2", "rekor://entry", "123"),
envelopeDigest: "sha256:envelope",
signedAt: DateTimeOffset.UnixEpoch),
ImmutableDictionary<string, string>.Empty);
public ValueTask<VexAttestationResponse> SignAsync(VexAttestationRequest request, CancellationToken cancellationToken)
{
LastRequest = request;
return ValueTask.FromResult(Response);
}
public ValueTask<VexAttestationVerification> VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken)
=> ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary<string, string>.Empty));
}
private sealed class RecordingCacheIndex : IVexCacheIndex
{
public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new();
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
=> ValueTask.FromResult<VexCacheEntry?>(null);
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
{
RemoveCalls[(signature.Value, format)] = true;
return ValueTask.CompletedTask;
}
}
private sealed class RecordingArtifactStore : IVexArtifactStore
{
public int SaveCount { get; private set; }
public ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken)
{
SaveCount++;
return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory", artifact.Content.Length, artifact.Metadata));
}
public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
=> ValueTask.FromResult<Stream?>(null);
}
private sealed class StaticPolicyEvaluator : IVexPolicyEvaluator
{
public StaticPolicyEvaluator(string version)
{
Version = version;
}
public string Version { get; }
public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default;
public double GetProviderWeight(VexProvider provider) => 1.0;
public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason)
{
rejectionReason = null;
return true;
}
}
private sealed class InMemoryExportDataSource : IVexExportDataSource
{
public ValueTask<VexExportDataSet> FetchAsync(VexQuery query, CancellationToken cancellationToken)
{
var claim = new VexClaim(
"CVE-2025-0001",
"vendor",
new VexProduct("pkg:demo/app", "Demo"),
VexClaimStatus.Affected,
new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:demo", new Uri("https://example.org/demo")),
DateTimeOffset.UtcNow,
DateTimeOffset.UtcNow);
var consensus = new VexConsensus(
"CVE-2025-0001",
claim.Product,
VexConsensusStatus.Affected,
DateTimeOffset.UtcNow,
new[] { new VexConsensusSource("vendor", VexClaimStatus.Affected, "sha256:demo", 1.0) },
conflicts: null,
policyVersion: "baseline/v1",
summary: "affected");
return ValueTask.FromResult(new VexExportDataSet(
ImmutableArray.Create(consensus),
ImmutableArray.Create(claim),
ImmutableArray.Create("vendor")));
}
}
private sealed class DummyExporter : IVexExporter
{
public DummyExporter(VexExportFormat format)
{
Format = format;
}
public VexExportFormat Format { get; }
public VexContentAddress Digest(VexExportRequest request)
=> new("sha256", "deadbeef");
public ValueTask<VexExportResult> SerializeAsync(VexExportRequest request, Stream output, CancellationToken cancellationToken)
{
var bytes = System.Text.Encoding.UTF8.GetBytes("{}");
output.Write(bytes);
return ValueTask.FromResult(new VexExportResult(new VexContentAddress("sha256", "deadbeef"), bytes.Length, ImmutableDictionary<string, string>.Empty));
}
}
}

View File

@@ -0,0 +1,33 @@
using System.Collections.Immutable;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export;
using System.IO.Abstractions.TestingHelpers;
namespace StellaOps.Excititor.Export.Tests;
public sealed class FileSystemArtifactStoreTests
{
[Fact]
public async Task SaveAsync_WritesArtifactToDisk()
{
var fs = new MockFileSystem();
var options = Options.Create(new FileSystemArtifactStoreOptions { RootPath = "/exports" });
var store = new FileSystemArtifactStore(options, NullLogger<FileSystemArtifactStore>.Instance, fs);
var content = new byte[] { 1, 2, 3 };
var artifact = new VexExportArtifact(
new VexContentAddress("sha256", "deadbeef"),
VexExportFormat.Json,
content,
ImmutableDictionary<string, string>.Empty);
var stored = await store.SaveAsync(artifact, CancellationToken.None);
Assert.Equal(artifact.Content.Length, stored.SizeBytes);
var filePath = fs.Path.Combine(options.Value.RootPath, stored.Location);
Assert.True(fs.FileExists(filePath));
Assert.Equal(content, fs.File.ReadAllBytes(filePath));
}
}

View File

@@ -0,0 +1,59 @@
using System.Collections.Immutable;
using System.IO.Abstractions.TestingHelpers;
using System.Linq;
using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export;
namespace StellaOps.Excititor.Export.Tests;
public sealed class OfflineBundleArtifactStoreTests
{
[Fact]
public async Task SaveAsync_WritesArtifactAndManifest()
{
var fs = new MockFileSystem();
var options = Options.Create(new OfflineBundleArtifactStoreOptions { RootPath = "/offline" });
var store = new OfflineBundleArtifactStore(options, NullLogger<OfflineBundleArtifactStore>.Instance, fs);
var content = new byte[] { 1, 2, 3 };
var digest = "sha256:" + Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(content)).ToLowerInvariant();
var artifact = new VexExportArtifact(
new VexContentAddress("sha256", digest.Split(':')[1]),
VexExportFormat.Json,
content,
ImmutableDictionary<string, string>.Empty);
var stored = await store.SaveAsync(artifact, CancellationToken.None);
var artifactPath = fs.Path.Combine(options.Value.RootPath, stored.Location);
Assert.True(fs.FileExists(artifactPath));
var manifestPath = fs.Path.Combine(options.Value.RootPath, options.Value.ManifestFileName);
Assert.True(fs.FileExists(manifestPath));
await using var manifestStream = fs.File.OpenRead(manifestPath);
using var document = await JsonDocument.ParseAsync(manifestStream);
var artifacts = document.RootElement.GetProperty("artifacts");
Assert.True(artifacts.GetArrayLength() >= 1);
var first = artifacts.EnumerateArray().First();
Assert.Equal(digest, first.GetProperty("digest").GetString());
}
[Fact]
public async Task SaveAsync_ThrowsOnDigestMismatch()
{
var fs = new MockFileSystem();
var options = Options.Create(new OfflineBundleArtifactStoreOptions { RootPath = "/offline" });
var store = new OfflineBundleArtifactStore(options, NullLogger<OfflineBundleArtifactStore>.Instance, fs);
var artifact = new VexExportArtifact(
new VexContentAddress("sha256", "deadbeef"),
VexExportFormat.Json,
new byte[] { 0x01, 0x02 },
ImmutableDictionary<string, string>.Empty);
await Assert.ThrowsAsync<InvalidOperationException>(() => store.SaveAsync(artifact, CancellationToken.None).AsTask());
}
}

View File

@@ -0,0 +1,95 @@
using System.Collections.Concurrent;
using System.Collections.Immutable;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export;
namespace StellaOps.Excititor.Export.Tests;
public sealed class S3ArtifactStoreTests
{
[Fact]
public async Task SaveAsync_UploadsContentWithMetadata()
{
var client = new FakeS3Client();
var options = Options.Create(new S3ArtifactStoreOptions { BucketName = "exports", Prefix = "vex" });
var store = new S3ArtifactStore(client, options, NullLogger<S3ArtifactStore>.Instance);
var content = new byte[] { 1, 2, 3, 4 };
var artifact = new VexExportArtifact(
new VexContentAddress("sha256", "deadbeef"),
VexExportFormat.Json,
content,
ImmutableDictionary<string, string>.Empty);
await store.SaveAsync(artifact, CancellationToken.None);
Assert.True(client.PutCalls.TryGetValue("exports", out var bucketEntries));
Assert.NotNull(bucketEntries);
var entry = bucketEntries!.Single();
Assert.Equal("vex/json/deadbeef.json", entry.Key);
Assert.Equal(content, entry.Content);
Assert.Equal("sha256:deadbeef", entry.Metadata["vex-digest"]);
}
[Fact]
public async Task OpenReadAsync_ReturnsStoredContent()
{
var client = new FakeS3Client();
var options = Options.Create(new S3ArtifactStoreOptions { BucketName = "exports", Prefix = "vex" });
var store = new S3ArtifactStore(client, options, NullLogger<S3ArtifactStore>.Instance);
var address = new VexContentAddress("sha256", "cafebabe");
client.SeedObject("exports", "vex/json/cafebabe.json", new byte[] { 9, 9, 9 });
var stream = await store.OpenReadAsync(address, CancellationToken.None);
Assert.NotNull(stream);
using var ms = new MemoryStream();
await stream!.CopyToAsync(ms);
Assert.Equal(new byte[] { 9, 9, 9 }, ms.ToArray());
}
private sealed class FakeS3Client : IS3ArtifactClient
{
public ConcurrentDictionary<string, List<S3Entry>> PutCalls { get; } = new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<(string Bucket, string Key), byte[]> _storage = new();
public void SeedObject(string bucket, string key, byte[] content)
{
PutCalls.GetOrAdd(bucket, _ => new List<S3Entry>()).Add(new S3Entry(key, content, new Dictionary<string, string>()));
_storage[(bucket, key)] = content;
}
public Task<bool> ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken)
=> Task.FromResult(_storage.ContainsKey((bucketName, key)));
public Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary<string, string> metadata, CancellationToken cancellationToken)
{
using var ms = new MemoryStream();
content.CopyTo(ms);
var bytes = ms.ToArray();
PutCalls.GetOrAdd(bucketName, _ => new List<S3Entry>()).Add(new S3Entry(key, bytes, new Dictionary<string, string>(metadata)));
_storage[(bucketName, key)] = bytes;
return Task.CompletedTask;
}
public Task<Stream?> GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken)
{
if (_storage.TryGetValue((bucketName, key), out var bytes))
{
return Task.FromResult<Stream?>(new MemoryStream(bytes, writable: false));
}
return Task.FromResult<Stream?>(null);
}
public Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken)
{
_storage.TryRemove((bucketName, key), out _);
return Task.CompletedTask;
}
public readonly record struct S3Entry(string Key, byte[] Content, IDictionary<string, string> Metadata);
}
}

View File

@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="20.0.28" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Export\StellaOps.Excititor.Export.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,82 @@
using Microsoft.Extensions.Logging.Abstractions;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Export.Tests;
public sealed class VexExportCacheServiceTests
{
[Fact]
public async Task InvalidateAsync_RemovesEntry()
{
var cacheIndex = new RecordingIndex();
var maintenance = new StubMaintenance();
var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance);
var signature = new VexQuerySignature("format=json|provider=vendor");
await service.InvalidateAsync(signature, VexExportFormat.Json, CancellationToken.None);
Assert.Equal(signature.Value, cacheIndex.LastSignature?.Value);
Assert.Equal(VexExportFormat.Json, cacheIndex.LastFormat);
Assert.Equal(1, cacheIndex.RemoveCalls);
}
[Fact]
public async Task PruneExpiredAsync_ReturnsCount()
{
var cacheIndex = new RecordingIndex();
var maintenance = new StubMaintenance { ExpiredCount = 3 };
var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance);
var removed = await service.PruneExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None);
Assert.Equal(3, removed);
}
[Fact]
public async Task PruneDanglingAsync_ReturnsCount()
{
var cacheIndex = new RecordingIndex();
var maintenance = new StubMaintenance { DanglingCount = 2 };
var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger<VexExportCacheService>.Instance);
var removed = await service.PruneDanglingAsync(CancellationToken.None);
Assert.Equal(2, removed);
}
private sealed class RecordingIndex : IVexCacheIndex
{
public VexQuerySignature? LastSignature { get; private set; }
public VexExportFormat LastFormat { get; private set; }
public int RemoveCalls { get; private set; }
public ValueTask<VexCacheEntry?> FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
=> ValueTask.FromResult<VexCacheEntry?>(null);
public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
{
LastSignature = signature;
LastFormat = format;
RemoveCalls++;
return ValueTask.CompletedTask;
}
}
private sealed class StubMaintenance : IVexCacheMaintenance
{
public int ExpiredCount { get; set; }
public int DanglingCount { get; set; }
public ValueTask<int> RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(ExpiredCount);
public ValueTask<int> RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null)
=> ValueTask.FromResult(DanglingCount);
}
}