Rename Vexer to Excititor

This commit is contained in:
2025-10-18 20:00:46 +03:00
parent fbd1826ef3
commit 7e1b10d3b2
263 changed files with 848 additions and 848 deletions

View File

@@ -0,0 +1,23 @@
# AGENTS
## Role
Produces deterministic VEX export artifacts, coordinates cache lookups, and bridges artifact storage with attestation generation.
## Scope
- Export orchestration pipeline: query signature resolution, cache lookup, snapshot building, attestation handoff.
- Format-neutral builder interfaces consumed by format-specific plug-ins.
- Artifact store abstraction wiring (S3/MinIO/filesystem) with offline-friendly packaging.
- Export metrics/logging and deterministic manifest emission.
## Participants
- WebService invokes the export engine to service `/excititor/export` requests.
- Attestation module receives built artifacts through this layer for signing.
- Worker reuses caching and artifact utilities for scheduled exports and GC routines.
## Interfaces & contracts
- `IExportEngine`, `IExportSnapshotBuilder`, cache provider interfaces, and artifact store adapters.
- Hook points for format plug-ins (JSON, JSONL, OpenVEX, CSAF, ZIP bundle).
## In/Out of scope
In: orchestration, caching, artifact store interactions, manifest metadata.
Out: format-specific serialization (lives in Formats.*), policy evaluation (Policy), HTTP presentation (WebService).
## Observability & security expectations
- Emit cache hit/miss counters, export durations, artifact sizes, and attestation timing logs.
- Ensure no sensitive tokens/URIs are logged.
## Tests
- Engine orchestration tests, cache behavior, and artifact lifecycle coverage will live in `../StellaOps.Excititor.Export.Tests`.

View File

@@ -0,0 +1,209 @@
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Export;
public interface IExportEngine
{
ValueTask<VexExportManifest> ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken);
}
public sealed record VexExportRequestContext(
VexQuery Query,
VexExportFormat Format,
DateTimeOffset RequestedAt,
bool ForceRefresh = false);
public interface IVexExportDataSource
{
ValueTask<VexExportDataSet> FetchAsync(VexQuery query, CancellationToken cancellationToken);
}
public sealed record VexExportDataSet(
ImmutableArray<VexConsensus> Consensus,
ImmutableArray<VexClaim> Claims,
ImmutableArray<string> SourceProviders);
public sealed class VexExportEngine : IExportEngine
{
private readonly IVexExportStore _exportStore;
private readonly IVexPolicyEvaluator _policyEvaluator;
private readonly IVexExportDataSource _dataSource;
private readonly IReadOnlyDictionary<VexExportFormat, IVexExporter> _exporters;
private readonly ILogger<VexExportEngine> _logger;
private readonly IVexCacheIndex? _cacheIndex;
private readonly IReadOnlyList<IVexArtifactStore> _artifactStores;
private readonly IVexAttestationClient? _attestationClient;
public VexExportEngine(
IVexExportStore exportStore,
IVexPolicyEvaluator policyEvaluator,
IVexExportDataSource dataSource,
IEnumerable<IVexExporter> exporters,
ILogger<VexExportEngine> logger,
IVexCacheIndex? cacheIndex = null,
IEnumerable<IVexArtifactStore>? artifactStores = null,
IVexAttestationClient? attestationClient = null)
{
_exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore));
_policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator));
_dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_cacheIndex = cacheIndex;
_artifactStores = artifactStores?.ToArray() ?? Array.Empty<IVexArtifactStore>();
_attestationClient = attestationClient;
if (exporters is null)
{
throw new ArgumentNullException(nameof(exporters));
}
_exporters = exporters.ToDictionary(x => x.Format);
}
public async ValueTask<VexExportManifest> ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(context);
var signature = VexQuerySignature.FromQuery(context.Query);
if (!context.ForceRefresh)
{
var cached = await _exportStore.FindAsync(signature, context.Format, cancellationToken).ConfigureAwait(false);
if (cached is not null)
{
_logger.LogInformation("Reusing cached export for {Signature} ({Format})", signature.Value, context.Format);
return new VexExportManifest(
cached.ExportId,
cached.QuerySignature,
cached.Format,
cached.CreatedAt,
cached.Artifact,
cached.ClaimCount,
cached.SourceProviders,
fromCache: true,
cached.ConsensusRevision,
cached.Attestation,
cached.SizeBytes);
}
}
else if (_cacheIndex is not null)
{
await _cacheIndex.RemoveAsync(signature, context.Format, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Force refresh requested; invalidated cache entry for {Signature} ({Format})", signature.Value, context.Format);
}
var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false);
var exporter = ResolveExporter(context.Format);
var exportRequest = new VexExportRequest(
context.Query,
dataset.Consensus,
dataset.Claims,
context.RequestedAt);
var digest = exporter.Digest(exportRequest);
var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}");
await using var buffer = new MemoryStream();
var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false);
if (_artifactStores.Count > 0)
{
var writtenBytes = buffer.ToArray();
try
{
var artifact = new VexExportArtifact(
result.Digest,
context.Format,
writtenBytes,
result.Metadata);
foreach (var store in _artifactStores)
{
await store.SaveAsync(artifact, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Stored export artifact {Digest} via {StoreCount} store(s)", result.Digest.ToUri(), _artifactStores.Count);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to store export artifact {Digest}", result.Digest.ToUri());
throw;
}
}
VexAttestationMetadata? attestationMetadata = null;
if (_attestationClient is not null)
{
var attestationRequest = new VexAttestationRequest(
exportId,
signature,
digest,
context.Format,
context.RequestedAt,
dataset.SourceProviders,
result.Metadata);
var response = await _attestationClient.SignAsync(attestationRequest, cancellationToken).ConfigureAwait(false);
attestationMetadata = response.Attestation;
if (!response.Diagnostics.IsEmpty)
{
foreach (var diagnostic in response.Diagnostics)
{
_logger.LogDebug(
"Attestation diagnostic {Key}={Value} for export {ExportId}",
diagnostic.Key,
diagnostic.Value,
exportId);
}
}
_logger.LogInformation("Attestation generated for export {ExportId}", exportId);
}
var manifest = new VexExportManifest(
exportId,
signature,
context.Format,
context.RequestedAt,
digest,
dataset.Claims.Length,
dataset.SourceProviders,
fromCache: false,
consensusRevision: _policyEvaluator.Version,
attestation: attestationMetadata,
sizeBytes: result.BytesWritten);
await _exportStore.SaveAsync(manifest, cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Export generated for {Signature} ({Format}) size={SizeBytes} bytes",
signature.Value,
context.Format,
result.BytesWritten);
return manifest;
}
private IVexExporter ResolveExporter(VexExportFormat format)
=> _exporters.TryGetValue(format, out var exporter)
? exporter
: throw new InvalidOperationException($"No exporter registered for format '{format}'.");
}
public static class VexExportServiceCollectionExtensions
{
public static IServiceCollection AddVexExportEngine(this IServiceCollection services)
{
services.AddSingleton<IExportEngine, VexExportEngine>();
services.AddVexExportCacheServices();
return services;
}
}

View File

@@ -0,0 +1,159 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Export;
public sealed class FileSystemArtifactStoreOptions
{
public string RootPath { get; set; } = ".";
public bool OverwriteExisting { get; set; } = false;
}
public sealed class FileSystemArtifactStore : IVexArtifactStore
{
private readonly IFileSystem _fileSystem;
private readonly FileSystemArtifactStoreOptions _options;
private readonly ILogger<FileSystemArtifactStore> _logger;
public FileSystemArtifactStore(
IOptions<FileSystemArtifactStoreOptions> options,
ILogger<FileSystemArtifactStore> logger,
IFileSystem? fileSystem = null)
{
ArgumentNullException.ThrowIfNull(options);
_options = options.Value;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_fileSystem = fileSystem ?? new FileSystem();
if (string.IsNullOrWhiteSpace(_options.RootPath))
{
throw new ArgumentException("RootPath must be provided for FileSystemArtifactStore.", nameof(options));
}
var root = _fileSystem.Path.GetFullPath(_options.RootPath);
_fileSystem.Directory.CreateDirectory(root);
_options.RootPath = root;
}
public async ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(artifact);
var relativePath = BuildArtifactPath(artifact.ContentAddress, artifact.Format);
var destination = _fileSystem.Path.Combine(_options.RootPath, relativePath);
var directory = _fileSystem.Path.GetDirectoryName(destination);
if (!string.IsNullOrEmpty(directory))
{
_fileSystem.Directory.CreateDirectory(directory);
}
if (_fileSystem.File.Exists(destination) && !_options.OverwriteExisting)
{
_logger.LogInformation("Artifact {Digest} already exists at {Path}; skipping write.", artifact.ContentAddress.ToUri(), destination);
}
else
{
await using var stream = _fileSystem.File.Create(destination);
await stream.WriteAsync(artifact.Content, cancellationToken).ConfigureAwait(false);
}
var location = destination.Replace(_options.RootPath, string.Empty).TrimStart(_fileSystem.Path.DirectorySeparatorChar, _fileSystem.Path.AltDirectorySeparatorChar);
return new VexStoredArtifact(
artifact.ContentAddress,
location,
artifact.Content.Length,
artifact.Metadata);
}
public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
{
var path = MaterializePath(contentAddress);
if (path is not null && _fileSystem.File.Exists(path))
{
_fileSystem.File.Delete(path);
}
return ValueTask.CompletedTask;
}
public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
{
var path = MaterializePath(contentAddress);
if (path is null || !_fileSystem.File.Exists(path))
{
return ValueTask.FromResult<Stream?>(null);
}
Stream stream = _fileSystem.File.OpenRead(path);
return ValueTask.FromResult<Stream?>(stream);
}
private static string BuildArtifactPath(VexContentAddress address, VexExportFormat format)
{
var formatSegment = format.ToString().ToLowerInvariant();
var safeDigest = address.Digest.Replace(':', '_');
var extension = GetExtension(format);
return Path.Combine(formatSegment, safeDigest + extension);
}
private string? MaterializePath(VexContentAddress address)
{
ArgumentNullException.ThrowIfNull(address);
var sanitized = address.Digest.Replace(':', '_');
foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat)))
{
var candidate = _fileSystem.Path.Combine(_options.RootPath, format.ToString().ToLowerInvariant(), sanitized + GetExtension(format));
if (_fileSystem.File.Exists(candidate))
{
return candidate;
}
}
// fallback: direct root search with common extensions
foreach (var extension in new[] { ".json", ".jsonl" })
{
var candidate = _fileSystem.Path.Combine(_options.RootPath, sanitized + extension);
if (_fileSystem.File.Exists(candidate))
{
return candidate;
}
}
return null;
}
private static string GetExtension(VexExportFormat format)
=> format switch
{
VexExportFormat.Json => ".json",
VexExportFormat.JsonLines => ".jsonl",
VexExportFormat.OpenVex => ".json",
VexExportFormat.Csaf => ".json",
_ => ".bin",
};
}
public static class FileSystemArtifactStoreServiceCollectionExtensions
{
public static IServiceCollection AddVexFileSystemArtifactStore(this IServiceCollection services, Action<FileSystemArtifactStoreOptions>? configure = null)
{
if (configure is not null)
{
services.Configure(configure);
}
services.AddSingleton<IVexArtifactStore, FileSystemArtifactStore>();
return services;
}
}

View File

@@ -0,0 +1,28 @@
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Export;
public sealed record VexExportArtifact(
VexContentAddress ContentAddress,
VexExportFormat Format,
ReadOnlyMemory<byte> Content,
IReadOnlyDictionary<string, string> Metadata);
public sealed record VexStoredArtifact(
VexContentAddress ContentAddress,
string Location,
long SizeBytes,
IReadOnlyDictionary<string, string> Metadata);
public interface IVexArtifactStore
{
ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken);
ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken);
ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken);
}

View File

@@ -0,0 +1,243 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.IO.Abstractions;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Export;
public sealed class OfflineBundleArtifactStoreOptions
{
public string RootPath { get; set; } = ".";
public string ArtifactsFolder { get; set; } = "artifacts";
public string BundlesFolder { get; set; } = "bundles";
public string ManifestFileName { get; set; } = "offline-manifest.json";
}
public sealed class OfflineBundleArtifactStore : IVexArtifactStore
{
private readonly IFileSystem _fileSystem;
private readonly OfflineBundleArtifactStoreOptions _options;
private readonly ILogger<OfflineBundleArtifactStore> _logger;
private readonly JsonSerializerOptions _serializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = true,
};
public OfflineBundleArtifactStore(
IOptions<OfflineBundleArtifactStoreOptions> options,
ILogger<OfflineBundleArtifactStore> logger,
IFileSystem? fileSystem = null)
{
ArgumentNullException.ThrowIfNull(options);
_options = options.Value;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_fileSystem = fileSystem ?? new FileSystem();
if (string.IsNullOrWhiteSpace(_options.RootPath))
{
throw new ArgumentException("RootPath must be provided for OfflineBundleArtifactStore.", nameof(options));
}
var root = _fileSystem.Path.GetFullPath(_options.RootPath);
_fileSystem.Directory.CreateDirectory(root);
_options.RootPath = root;
_fileSystem.Directory.CreateDirectory(GetArtifactsRoot());
_fileSystem.Directory.CreateDirectory(GetBundlesRoot());
}
public async ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(artifact);
EnforceDigestMatch(artifact);
var artifactRelativePath = BuildArtifactRelativePath(artifact);
var artifactFullPath = _fileSystem.Path.Combine(_options.RootPath, artifactRelativePath);
var artifactDirectory = _fileSystem.Path.GetDirectoryName(artifactFullPath);
if (!string.IsNullOrEmpty(artifactDirectory))
{
_fileSystem.Directory.CreateDirectory(artifactDirectory);
}
await using (var stream = _fileSystem.File.Create(artifactFullPath))
{
await stream.WriteAsync(artifact.Content, cancellationToken).ConfigureAwait(false);
}
WriteOfflineBundle(artifactRelativePath, artifact, cancellationToken);
await UpdateManifestAsync(artifactRelativePath, artifact, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Stored offline artifact {Digest} at {Path}", artifact.ContentAddress.ToUri(), artifactRelativePath);
return new VexStoredArtifact(
artifact.ContentAddress,
artifactRelativePath,
artifact.Content.Length,
artifact.Metadata);
}
public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(contentAddress);
var sanitized = contentAddress.Digest.Replace(':', '_');
var artifactsRoot = GetArtifactsRoot();
foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat)))
{
var extension = GetExtension(format);
var path = _fileSystem.Path.Combine(artifactsRoot, format.ToString().ToLowerInvariant(), sanitized + extension);
if (_fileSystem.File.Exists(path))
{
_fileSystem.File.Delete(path);
}
var bundlePath = _fileSystem.Path.Combine(GetBundlesRoot(), sanitized + ".zip");
if (_fileSystem.File.Exists(bundlePath))
{
_fileSystem.File.Delete(bundlePath);
}
}
return ValueTask.CompletedTask;
}
public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(contentAddress);
var artifactsRoot = GetArtifactsRoot();
var sanitized = contentAddress.Digest.Replace(':', '_');
foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat)))
{
var candidate = _fileSystem.Path.Combine(artifactsRoot, format.ToString().ToLowerInvariant(), sanitized + GetExtension(format));
if (_fileSystem.File.Exists(candidate))
{
return ValueTask.FromResult<Stream?>(_fileSystem.File.OpenRead(candidate));
}
}
return ValueTask.FromResult<Stream?>(null);
}
private void EnforceDigestMatch(VexExportArtifact artifact)
{
if (!artifact.ContentAddress.Algorithm.Equals("sha256", StringComparison.OrdinalIgnoreCase))
{
return;
}
using var sha = SHA256.Create();
var computed = "sha256:" + Convert.ToHexString(sha.ComputeHash(artifact.Content.ToArray())).ToLowerInvariant();
if (!string.Equals(computed, artifact.ContentAddress.ToUri(), StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException($"Artifact content digest mismatch. Expected {artifact.ContentAddress.ToUri()} but computed {computed}.");
}
}
private string BuildArtifactRelativePath(VexExportArtifact artifact)
{
var sanitized = artifact.ContentAddress.Digest.Replace(':', '_');
var folder = _fileSystem.Path.Combine(_options.ArtifactsFolder, artifact.Format.ToString().ToLowerInvariant());
return _fileSystem.Path.Combine(folder, sanitized + GetExtension(artifact.Format));
}
private void WriteOfflineBundle(string artifactRelativePath, VexExportArtifact artifact, CancellationToken cancellationToken)
{
var zipPath = _fileSystem.Path.Combine(GetBundlesRoot(), artifact.ContentAddress.Digest.Replace(':', '_') + ".zip");
using var zipStream = _fileSystem.File.Create(zipPath);
using var archive = new ZipArchive(zipStream, ZipArchiveMode.Create);
var entry = archive.CreateEntry(artifactRelativePath, CompressionLevel.Optimal);
using (var entryStream = entry.Open())
{
entryStream.Write(artifact.Content.Span);
}
// embed metadata file
var metadataEntry = archive.CreateEntry("metadata.json", CompressionLevel.Optimal);
using var metadataStream = new StreamWriter(metadataEntry.Open());
var metadata = new Dictionary<string, object?>
{
["digest"] = artifact.ContentAddress.ToUri(),
["format"] = artifact.Format.ToString().ToLowerInvariant(),
["sizeBytes"] = artifact.Content.Length,
["metadata"] = artifact.Metadata,
};
metadataStream.Write(JsonSerializer.Serialize(metadata, _serializerOptions));
}
private async Task UpdateManifestAsync(string artifactRelativePath, VexExportArtifact artifact, CancellationToken cancellationToken)
{
var manifestPath = _fileSystem.Path.Combine(_options.RootPath, _options.ManifestFileName);
var records = new List<ManifestEntry>();
if (_fileSystem.File.Exists(manifestPath))
{
await using var existingStream = _fileSystem.File.OpenRead(manifestPath);
var existing = await JsonSerializer.DeserializeAsync<ManifestDocument>(existingStream, _serializerOptions, cancellationToken).ConfigureAwait(false);
if (existing is not null)
{
records.AddRange(existing.Artifacts);
}
}
records.RemoveAll(x => string.Equals(x.Digest, artifact.ContentAddress.ToUri(), StringComparison.OrdinalIgnoreCase));
records.Add(new ManifestEntry(
artifact.ContentAddress.ToUri(),
artifact.Format.ToString().ToLowerInvariant(),
artifactRelativePath.Replace("\\", "/"),
artifact.Content.Length,
artifact.Metadata));
records.Sort(static (a, b) => string.CompareOrdinal(a.Digest, b.Digest));
var doc = new ManifestDocument(records.ToImmutableArray());
await using var stream = _fileSystem.File.Create(manifestPath);
await JsonSerializer.SerializeAsync(stream, doc, _serializerOptions, cancellationToken).ConfigureAwait(false);
}
private string GetArtifactsRoot() => _fileSystem.Path.Combine(_options.RootPath, _options.ArtifactsFolder);
private string GetBundlesRoot() => _fileSystem.Path.Combine(_options.RootPath, _options.BundlesFolder);
private static string GetExtension(VexExportFormat format)
=> format switch
{
VexExportFormat.Json => ".json",
VexExportFormat.JsonLines => ".jsonl",
VexExportFormat.OpenVex => ".json",
VexExportFormat.Csaf => ".json",
_ => ".bin",
};
private sealed record ManifestDocument(ImmutableArray<ManifestEntry> Artifacts);
private sealed record ManifestEntry(string Digest, string Format, string Path, long SizeBytes, IReadOnlyDictionary<string, string> Metadata);
}
public static class OfflineBundleArtifactStoreServiceCollectionExtensions
{
public static IServiceCollection AddVexOfflineBundleArtifactStore(this IServiceCollection services, Action<OfflineBundleArtifactStoreOptions>? configure = null)
{
if (configure is not null)
{
services.Configure(configure);
}
services.AddSingleton<IVexArtifactStore, OfflineBundleArtifactStore>();
return services;
}
}

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Excititor.Export.Tests")]

View File

@@ -0,0 +1,181 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Core;
namespace StellaOps.Excititor.Export;
public sealed class S3ArtifactStoreOptions
{
public string BucketName { get; set; } = string.Empty;
public string? Prefix { get; set; }
= null;
public bool OverwriteExisting { get; set; }
= true;
}
public interface IS3ArtifactClient
{
Task<bool> ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken);
Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary<string, string> metadata, CancellationToken cancellationToken);
Task<Stream?> GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken);
Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken);
}
public sealed class S3ArtifactStore : IVexArtifactStore
{
private readonly IS3ArtifactClient _client;
private readonly S3ArtifactStoreOptions _options;
private readonly ILogger<S3ArtifactStore> _logger;
public S3ArtifactStore(
IS3ArtifactClient client,
IOptions<S3ArtifactStoreOptions> options,
ILogger<S3ArtifactStore> logger)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
ArgumentNullException.ThrowIfNull(options);
_options = options.Value;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (string.IsNullOrWhiteSpace(_options.BucketName))
{
throw new ArgumentException("BucketName must be provided for S3ArtifactStore.", nameof(options));
}
}
public async ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(artifact);
var key = BuildObjectKey(artifact.ContentAddress, artifact.Format);
if (!_options.OverwriteExisting)
{
var exists = await _client.ObjectExistsAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false);
if (exists)
{
_logger.LogInformation("S3 object {Bucket}/{Key} already exists; skipping upload.", _options.BucketName, key);
return new VexStoredArtifact(artifact.ContentAddress, key, artifact.Content.Length, artifact.Metadata);
}
}
using var contentStream = new MemoryStream(artifact.Content.ToArray());
await _client.PutObjectAsync(
_options.BucketName,
key,
contentStream,
BuildObjectMetadata(artifact),
cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Uploaded export artifact {Digest} to {Bucket}/{Key}", artifact.ContentAddress.ToUri(), _options.BucketName, key);
return new VexStoredArtifact(
artifact.ContentAddress,
key,
artifact.Content.Length,
artifact.Metadata);
}
public async ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(contentAddress);
foreach (var key in BuildCandidateKeys(contentAddress))
{
await _client.DeleteObjectAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation("Deleted export artifact {Digest} from {Bucket}", contentAddress.ToUri(), _options.BucketName);
}
public async ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(contentAddress);
foreach (var key in BuildCandidateKeys(contentAddress))
{
var stream = await _client.GetObjectAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false);
if (stream is not null)
{
return stream;
}
}
return null;
}
private string BuildObjectKey(VexContentAddress address, VexExportFormat format)
{
var sanitizedDigest = address.Digest.Replace(':', '_');
var prefix = string.IsNullOrWhiteSpace(_options.Prefix) ? string.Empty : _options.Prefix.TrimEnd('/') + "/";
var formatSegment = format.ToString().ToLowerInvariant();
return $"{prefix}{formatSegment}/{sanitizedDigest}{GetExtension(format)}";
}
private IEnumerable<string> BuildCandidateKeys(VexContentAddress address)
{
foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat)))
{
yield return BuildObjectKey(address, format);
}
if (!string.IsNullOrWhiteSpace(_options.Prefix))
{
yield return $"{_options.Prefix.TrimEnd('/')}/{address.Digest.Replace(':', '_')}";
}
yield return address.Digest.Replace(':', '_');
}
private static IDictionary<string, string> BuildObjectMetadata(VexExportArtifact artifact)
{
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["vex-format"] = artifact.Format.ToString().ToLowerInvariant(),
["vex-digest"] = artifact.ContentAddress.ToUri(),
["content-type"] = artifact.Format switch
{
VexExportFormat.Json => "application/json",
VexExportFormat.JsonLines => "application/json",
VexExportFormat.OpenVex => "application/vnd.openvex+json",
VexExportFormat.Csaf => "application/json",
_ => "application/octet-stream",
},
};
foreach (var kvp in artifact.Metadata)
{
metadata[$"meta-{kvp.Key}"] = kvp.Value;
}
return metadata;
}
private static string GetExtension(VexExportFormat format)
=> format switch
{
VexExportFormat.Json => ".json",
VexExportFormat.JsonLines => ".jsonl",
VexExportFormat.OpenVex => ".json",
VexExportFormat.Csaf => ".json",
_ => ".bin",
};
}
public static class S3ArtifactStoreServiceCollectionExtensions
{
public static IServiceCollection AddVexS3ArtifactStore(this IServiceCollection services, Action<S3ArtifactStoreOptions> configure)
{
ArgumentNullException.ThrowIfNull(configure);
services.Configure(configure);
services.AddSingleton<IVexArtifactStore, S3ArtifactStore>();
return services;
}
}

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" />
<PackageReference Include="System.IO.Abstractions" Version="20.0.28" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Policy\StellaOps.Excititor.Policy.csproj" />
<ProjectReference Include="..\StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,9 @@
If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md).
# TASKS
| Task | Owner(s) | Depends on | Notes |
|---|---|---|---|
|EXCITITOR-EXPORT-01-001 Export engine orchestration|Team Excititor Export|EXCITITOR-CORE-01-003|DONE (2025-10-15) Export engine scaffolding with cache lookup, data source hooks, and deterministic manifest emission.|
|EXCITITOR-EXPORT-01-002 Cache index & eviction hooks|Team Excititor Export|EXCITITOR-EXPORT-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-16)** Export engine now invalidates cache entries on force refresh, cache services expose prune/invalidate APIs, and storage maintenance trims expired/dangling records with Mongo2Go coverage.|
|EXCITITOR-EXPORT-01-003 Artifact store adapters|Team Excititor Export|EXCITITOR-EXPORT-01-001|**DONE (2025-10-16)** Implemented multi-store pipeline with filesystem, S3-compatible, and offline bundle adapters (hash verification + manifest/zip output) plus unit coverage and DI hooks.|
|EXCITITOR-EXPORT-01-004 Attestation handoff integration|Team Excititor Export|EXCITITOR-EXPORT-01-001, EXCITITOR-ATTEST-01-001|**DONE (2025-10-17)** Export engine now invokes attestation client, logs diagnostics, and persists Rekor/envelope metadata on manifests; regression coverage added in `ExportEngineTests.ExportAsync_AttachesAttestationMetadata`.|
|EXCITITOR-EXPORT-01-005 Score & resolve envelope surfaces|Team Excititor Export|EXCITITOR-EXPORT-01-004, EXCITITOR-CORE-02-001|TODO Emit consensus+score envelopes in export manifests, include policy/scoring digests, and update offline bundle/ORAS layouts to carry signed VEX responses.|

View File

@@ -0,0 +1,54 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Storage.Mongo;
namespace StellaOps.Excititor.Export;
public interface IVexExportCacheService
{
ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken);
ValueTask<int> PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken);
ValueTask<int> PruneDanglingAsync(CancellationToken cancellationToken);
}
internal sealed class VexExportCacheService : IVexExportCacheService
{
private readonly IVexCacheIndex _cacheIndex;
private readonly IVexCacheMaintenance _maintenance;
private readonly ILogger<VexExportCacheService> _logger;
public VexExportCacheService(
IVexCacheIndex cacheIndex,
IVexCacheMaintenance maintenance,
ILogger<VexExportCacheService> logger)
{
_cacheIndex = cacheIndex ?? throw new ArgumentNullException(nameof(cacheIndex));
_maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(signature);
await _cacheIndex.RemoveAsync(signature, format, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Invalidated export cache entry {Signature} ({Format})", signature.Value, format);
}
public ValueTask<int> PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
=> _maintenance.RemoveExpiredAsync(asOf, cancellationToken);
public ValueTask<int> PruneDanglingAsync(CancellationToken cancellationToken)
=> _maintenance.RemoveMissingManifestReferencesAsync(cancellationToken);
}
public static class VexExportCacheServiceCollectionExtensions
{
public static IServiceCollection AddVexExportCacheServices(this IServiceCollection services)
{
services.AddSingleton<IVexExportCacheService, VexExportCacheService>();
return services;
}
}