commit
	
		
			
	
		
	
	
		
	
		
			Some checks failed
		
		
	
	
		
			
				
	
				Build Test Deploy / build-test (push) Has been cancelled
				
			
		
			
				
	
				Build Test Deploy / authority-container (push) Has been cancelled
				
			
		
			
				
	
				Build Test Deploy / docs (push) Has been cancelled
				
			
		
			
				
	
				Build Test Deploy / deploy (push) Has been cancelled
				
			
		
			
				
	
				Docs CI / lint-and-preview (push) Has been cancelled
				
			
		
		
	
	
				
					
				
			
		
			Some checks failed
		
		
	
	Build Test Deploy / build-test (push) Has been cancelled
				
			Build Test Deploy / authority-container (push) Has been cancelled
				
			Build Test Deploy / docs (push) Has been cancelled
				
			Build Test Deploy / deploy (push) Has been cancelled
				
			Docs CI / lint-and-preview (push) Has been cancelled
				
			This commit is contained in:
		| @@ -37,18 +37,24 @@ public sealed class VexExportEngine : IExportEngine | ||||
|     private readonly IVexExportDataSource _dataSource; | ||||
|     private readonly IReadOnlyDictionary<VexExportFormat, IVexExporter> _exporters; | ||||
|     private readonly ILogger<VexExportEngine> _logger; | ||||
|     private readonly IVexCacheIndex? _cacheIndex; | ||||
|     private readonly IReadOnlyList<IVexArtifactStore> _artifactStores; | ||||
|  | ||||
|     public VexExportEngine( | ||||
|         IVexExportStore exportStore, | ||||
|         IVexPolicyEvaluator policyEvaluator, | ||||
|         IVexExportDataSource dataSource, | ||||
|         IEnumerable<IVexExporter> exporters, | ||||
|         ILogger<VexExportEngine> logger) | ||||
|         ILogger<VexExportEngine> logger, | ||||
|         IVexCacheIndex? cacheIndex = null, | ||||
|         IEnumerable<IVexArtifactStore>? artifactStores = null) | ||||
|     { | ||||
|         _exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore)); | ||||
|         _policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator)); | ||||
|         _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|         _cacheIndex = cacheIndex; | ||||
|         _artifactStores = artifactStores?.ToArray() ?? Array.Empty<IVexArtifactStore>(); | ||||
|  | ||||
|         if (exporters is null) | ||||
|         { | ||||
| @@ -69,9 +75,25 @@ public sealed class VexExportEngine : IExportEngine | ||||
|             if (cached is not null) | ||||
|             { | ||||
|                 _logger.LogInformation("Reusing cached export for {Signature} ({Format})", signature.Value, context.Format); | ||||
|                 return cached with { FromCache = true }; | ||||
|                 return new VexExportManifest( | ||||
|                     cached.ExportId, | ||||
|                     cached.QuerySignature, | ||||
|                     cached.Format, | ||||
|                     cached.CreatedAt, | ||||
|                     cached.Artifact, | ||||
|                     cached.ClaimCount, | ||||
|                     cached.SourceProviders, | ||||
|                     fromCache: true, | ||||
|                     cached.ConsensusRevision, | ||||
|                     cached.Attestation, | ||||
|                     cached.SizeBytes); | ||||
|             } | ||||
|         } | ||||
|         else if (_cacheIndex is not null) | ||||
|         { | ||||
|             await _cacheIndex.RemoveAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); | ||||
|             _logger.LogInformation("Force refresh requested; invalidated cache entry for {Signature} ({Format})", signature.Value, context.Format); | ||||
|         } | ||||
|  | ||||
|         var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false); | ||||
|         var exporter = ResolveExporter(context.Format); | ||||
| @@ -87,6 +109,31 @@ public sealed class VexExportEngine : IExportEngine | ||||
|         await using var buffer = new MemoryStream(); | ||||
|         var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         if (_artifactStores.Count > 0) | ||||
|         { | ||||
|             var writtenBytes = buffer.ToArray(); | ||||
|             try | ||||
|             { | ||||
|                 var artifact = new VexExportArtifact( | ||||
|                     result.Digest, | ||||
|                     context.Format, | ||||
|                     writtenBytes, | ||||
|                     result.Metadata); | ||||
|  | ||||
|                 foreach (var store in _artifactStores) | ||||
|                 { | ||||
|                     await store.SaveAsync(artifact, cancellationToken).ConfigureAwait(false); | ||||
|                 } | ||||
|  | ||||
|                 _logger.LogInformation("Stored export artifact {Digest} via {StoreCount} store(s)", result.Digest.ToUri(), _artifactStores.Count); | ||||
|             } | ||||
|             catch (Exception ex) | ||||
|             { | ||||
|                 _logger.LogError(ex, "Failed to store export artifact {Digest}", result.Digest.ToUri()); | ||||
|                 throw; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}"); | ||||
|         var manifest = new VexExportManifest( | ||||
|             exportId, | ||||
| @@ -123,6 +170,7 @@ public static class VexExportServiceCollectionExtensions | ||||
|     public static IServiceCollection AddVexExportEngine(this IServiceCollection services) | ||||
|     { | ||||
|         services.AddSingleton<IExportEngine, VexExportEngine>(); | ||||
|         services.AddVexExportCacheServices(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
|   | ||||
							
								
								
									
										159
									
								
								src/StellaOps.Vexer.Export/FileSystemArtifactStore.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										159
									
								
								src/StellaOps.Vexer.Export/FileSystemArtifactStore.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,159 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using System.IO.Abstractions; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Vexer.Core; | ||||
|  | ||||
| namespace StellaOps.Vexer.Export; | ||||
|  | ||||
| public sealed class FileSystemArtifactStoreOptions | ||||
| { | ||||
|     public string RootPath { get; set; } = "."; | ||||
|  | ||||
|     public bool OverwriteExisting { get; set; } = false; | ||||
| } | ||||
|  | ||||
| public sealed class FileSystemArtifactStore : IVexArtifactStore | ||||
| { | ||||
|     private readonly IFileSystem _fileSystem; | ||||
|     private readonly FileSystemArtifactStoreOptions _options; | ||||
|     private readonly ILogger<FileSystemArtifactStore> _logger; | ||||
|  | ||||
|     public FileSystemArtifactStore( | ||||
|         IOptions<FileSystemArtifactStoreOptions> options, | ||||
|         ILogger<FileSystemArtifactStore> logger, | ||||
|         IFileSystem? fileSystem = null) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(options); | ||||
|         _options = options.Value; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|         _fileSystem = fileSystem ?? new FileSystem(); | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(_options.RootPath)) | ||||
|         { | ||||
|             throw new ArgumentException("RootPath must be provided for FileSystemArtifactStore.", nameof(options)); | ||||
|         } | ||||
|  | ||||
|         var root = _fileSystem.Path.GetFullPath(_options.RootPath); | ||||
|         _fileSystem.Directory.CreateDirectory(root); | ||||
|         _options.RootPath = root; | ||||
|     } | ||||
|  | ||||
|     public async ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(artifact); | ||||
|  | ||||
|         var relativePath = BuildArtifactPath(artifact.ContentAddress, artifact.Format); | ||||
|         var destination = _fileSystem.Path.Combine(_options.RootPath, relativePath); | ||||
|         var directory = _fileSystem.Path.GetDirectoryName(destination); | ||||
|         if (!string.IsNullOrEmpty(directory)) | ||||
|         { | ||||
|             _fileSystem.Directory.CreateDirectory(directory); | ||||
|         } | ||||
|  | ||||
|         if (_fileSystem.File.Exists(destination) && !_options.OverwriteExisting) | ||||
|         { | ||||
|             _logger.LogInformation("Artifact {Digest} already exists at {Path}; skipping write.", artifact.ContentAddress.ToUri(), destination); | ||||
|         } | ||||
|         else | ||||
|         { | ||||
|             await using var stream = _fileSystem.File.Create(destination); | ||||
|             await stream.WriteAsync(artifact.Content, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|  | ||||
|         var location = destination.Replace(_options.RootPath, string.Empty).TrimStart(_fileSystem.Path.DirectorySeparatorChar, _fileSystem.Path.AltDirectorySeparatorChar); | ||||
|  | ||||
|         return new VexStoredArtifact( | ||||
|             artifact.ContentAddress, | ||||
|             location, | ||||
|             artifact.Content.Length, | ||||
|             artifact.Metadata); | ||||
|     } | ||||
|  | ||||
|     public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var path = MaterializePath(contentAddress); | ||||
|         if (path is not null && _fileSystem.File.Exists(path)) | ||||
|         { | ||||
|             _fileSystem.File.Delete(path); | ||||
|         } | ||||
|  | ||||
|         return ValueTask.CompletedTask; | ||||
|     } | ||||
|  | ||||
|     public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var path = MaterializePath(contentAddress); | ||||
|         if (path is null || !_fileSystem.File.Exists(path)) | ||||
|         { | ||||
|             return ValueTask.FromResult<Stream?>(null); | ||||
|         } | ||||
|  | ||||
|         Stream stream = _fileSystem.File.OpenRead(path); | ||||
|         return ValueTask.FromResult<Stream?>(stream); | ||||
|     } | ||||
|  | ||||
|     private static string BuildArtifactPath(VexContentAddress address, VexExportFormat format) | ||||
|     { | ||||
|         var formatSegment = format.ToString().ToLowerInvariant(); | ||||
|         var safeDigest = address.Digest.Replace(':', '_'); | ||||
|         var extension = GetExtension(format); | ||||
|         return Path.Combine(formatSegment, safeDigest + extension); | ||||
|     } | ||||
|  | ||||
|     private string? MaterializePath(VexContentAddress address) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(address); | ||||
|         var sanitized = address.Digest.Replace(':', '_'); | ||||
|  | ||||
|         foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) | ||||
|         { | ||||
|             var candidate = _fileSystem.Path.Combine(_options.RootPath, format.ToString().ToLowerInvariant(), sanitized + GetExtension(format)); | ||||
|             if (_fileSystem.File.Exists(candidate)) | ||||
|             { | ||||
|                 return candidate; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         // fallback: direct root search with common extensions | ||||
|         foreach (var extension in new[] { ".json", ".jsonl" }) | ||||
|         { | ||||
|             var candidate = _fileSystem.Path.Combine(_options.RootPath, sanitized + extension); | ||||
|             if (_fileSystem.File.Exists(candidate)) | ||||
|             { | ||||
|                 return candidate; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private static string GetExtension(VexExportFormat format) | ||||
|         => format switch | ||||
|         { | ||||
|             VexExportFormat.Json => ".json", | ||||
|             VexExportFormat.JsonLines => ".jsonl", | ||||
|             VexExportFormat.OpenVex => ".json", | ||||
|             VexExportFormat.Csaf => ".json", | ||||
|             _ => ".bin", | ||||
|         }; | ||||
| } | ||||
|  | ||||
| public static class FileSystemArtifactStoreServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddVexFileSystemArtifactStore(this IServiceCollection services, Action<FileSystemArtifactStoreOptions>? configure = null) | ||||
|     { | ||||
|         if (configure is not null) | ||||
|         { | ||||
|             services.Configure(configure); | ||||
|         } | ||||
|  | ||||
|         services.AddSingleton<IVexArtifactStore, FileSystemArtifactStore>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										28
									
								
								src/StellaOps.Vexer.Export/IVexArtifactStore.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										28
									
								
								src/StellaOps.Vexer.Export/IVexArtifactStore.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,28 @@ | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using StellaOps.Vexer.Core; | ||||
|  | ||||
| namespace StellaOps.Vexer.Export; | ||||
|  | ||||
| public sealed record VexExportArtifact( | ||||
|     VexContentAddress ContentAddress, | ||||
|     VexExportFormat Format, | ||||
|     ReadOnlyMemory<byte> Content, | ||||
|     IReadOnlyDictionary<string, string> Metadata); | ||||
|  | ||||
| public sealed record VexStoredArtifact( | ||||
|     VexContentAddress ContentAddress, | ||||
|     string Location, | ||||
|     long SizeBytes, | ||||
|     IReadOnlyDictionary<string, string> Metadata); | ||||
|  | ||||
| public interface IVexArtifactStore | ||||
| { | ||||
|     ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken); | ||||
|  | ||||
|     ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken); | ||||
|  | ||||
|     ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken); | ||||
| } | ||||
							
								
								
									
										243
									
								
								src/StellaOps.Vexer.Export/OfflineBundleArtifactStore.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										243
									
								
								src/StellaOps.Vexer.Export/OfflineBundleArtifactStore.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,243 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.Collections.Immutable; | ||||
| using System.IO; | ||||
| using System.IO.Abstractions; | ||||
| using System.IO.Compression; | ||||
| using System.Security.Cryptography; | ||||
| using System.Text.Json; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Vexer.Core; | ||||
|  | ||||
| namespace StellaOps.Vexer.Export; | ||||
|  | ||||
| public sealed class OfflineBundleArtifactStoreOptions | ||||
| { | ||||
|     public string RootPath { get; set; } = "."; | ||||
|  | ||||
|     public string ArtifactsFolder { get; set; } = "artifacts"; | ||||
|  | ||||
|     public string BundlesFolder { get; set; } = "bundles"; | ||||
|  | ||||
|     public string ManifestFileName { get; set; } = "offline-manifest.json"; | ||||
| } | ||||
|  | ||||
| public sealed class OfflineBundleArtifactStore : IVexArtifactStore | ||||
| { | ||||
|     private readonly IFileSystem _fileSystem; | ||||
|     private readonly OfflineBundleArtifactStoreOptions _options; | ||||
|     private readonly ILogger<OfflineBundleArtifactStore> _logger; | ||||
|     private readonly JsonSerializerOptions _serializerOptions = new() | ||||
|     { | ||||
|         PropertyNamingPolicy = JsonNamingPolicy.CamelCase, | ||||
|         WriteIndented = true, | ||||
|     }; | ||||
|  | ||||
|     public OfflineBundleArtifactStore( | ||||
|         IOptions<OfflineBundleArtifactStoreOptions> options, | ||||
|         ILogger<OfflineBundleArtifactStore> logger, | ||||
|         IFileSystem? fileSystem = null) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(options); | ||||
|         _options = options.Value; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|         _fileSystem = fileSystem ?? new FileSystem(); | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(_options.RootPath)) | ||||
|         { | ||||
|             throw new ArgumentException("RootPath must be provided for OfflineBundleArtifactStore.", nameof(options)); | ||||
|         } | ||||
|  | ||||
|         var root = _fileSystem.Path.GetFullPath(_options.RootPath); | ||||
|         _fileSystem.Directory.CreateDirectory(root); | ||||
|         _options.RootPath = root; | ||||
|         _fileSystem.Directory.CreateDirectory(GetArtifactsRoot()); | ||||
|         _fileSystem.Directory.CreateDirectory(GetBundlesRoot()); | ||||
|     } | ||||
|  | ||||
|     public async ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(artifact); | ||||
|         EnforceDigestMatch(artifact); | ||||
|  | ||||
|         var artifactRelativePath = BuildArtifactRelativePath(artifact); | ||||
|         var artifactFullPath = _fileSystem.Path.Combine(_options.RootPath, artifactRelativePath); | ||||
|         var artifactDirectory = _fileSystem.Path.GetDirectoryName(artifactFullPath); | ||||
|         if (!string.IsNullOrEmpty(artifactDirectory)) | ||||
|         { | ||||
|             _fileSystem.Directory.CreateDirectory(artifactDirectory); | ||||
|         } | ||||
|  | ||||
|         await using (var stream = _fileSystem.File.Create(artifactFullPath)) | ||||
|         { | ||||
|             await stream.WriteAsync(artifact.Content, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|  | ||||
|         WriteOfflineBundle(artifactRelativePath, artifact, cancellationToken); | ||||
|         await UpdateManifestAsync(artifactRelativePath, artifact, cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         _logger.LogInformation("Stored offline artifact {Digest} at {Path}", artifact.ContentAddress.ToUri(), artifactRelativePath); | ||||
|  | ||||
|         return new VexStoredArtifact( | ||||
|             artifact.ContentAddress, | ||||
|             artifactRelativePath, | ||||
|             artifact.Content.Length, | ||||
|             artifact.Metadata); | ||||
|     } | ||||
|  | ||||
|     public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(contentAddress); | ||||
|         var sanitized = contentAddress.Digest.Replace(':', '_'); | ||||
|         var artifactsRoot = GetArtifactsRoot(); | ||||
|  | ||||
|         foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) | ||||
|         { | ||||
|             var extension = GetExtension(format); | ||||
|             var path = _fileSystem.Path.Combine(artifactsRoot, format.ToString().ToLowerInvariant(), sanitized + extension); | ||||
|             if (_fileSystem.File.Exists(path)) | ||||
|             { | ||||
|                 _fileSystem.File.Delete(path); | ||||
|             } | ||||
|  | ||||
|             var bundlePath = _fileSystem.Path.Combine(GetBundlesRoot(), sanitized + ".zip"); | ||||
|             if (_fileSystem.File.Exists(bundlePath)) | ||||
|             { | ||||
|                 _fileSystem.File.Delete(bundlePath); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return ValueTask.CompletedTask; | ||||
|     } | ||||
|  | ||||
|     public ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(contentAddress); | ||||
|         var artifactsRoot = GetArtifactsRoot(); | ||||
|         var sanitized = contentAddress.Digest.Replace(':', '_'); | ||||
|  | ||||
|         foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) | ||||
|         { | ||||
|             var candidate = _fileSystem.Path.Combine(artifactsRoot, format.ToString().ToLowerInvariant(), sanitized + GetExtension(format)); | ||||
|             if (_fileSystem.File.Exists(candidate)) | ||||
|             { | ||||
|                 return ValueTask.FromResult<Stream?>(_fileSystem.File.OpenRead(candidate)); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return ValueTask.FromResult<Stream?>(null); | ||||
|     } | ||||
|  | ||||
|     private void EnforceDigestMatch(VexExportArtifact artifact) | ||||
|     { | ||||
|         if (!artifact.ContentAddress.Algorithm.Equals("sha256", StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             return; | ||||
|         } | ||||
|  | ||||
|         using var sha = SHA256.Create(); | ||||
|         var computed = "sha256:" + Convert.ToHexString(sha.ComputeHash(artifact.Content.ToArray())).ToLowerInvariant(); | ||||
|         if (!string.Equals(computed, artifact.ContentAddress.ToUri(), StringComparison.OrdinalIgnoreCase)) | ||||
|         { | ||||
|             throw new InvalidOperationException($"Artifact content digest mismatch. Expected {artifact.ContentAddress.ToUri()} but computed {computed}."); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     private string BuildArtifactRelativePath(VexExportArtifact artifact) | ||||
|     { | ||||
|         var sanitized = artifact.ContentAddress.Digest.Replace(':', '_'); | ||||
|         var folder = _fileSystem.Path.Combine(_options.ArtifactsFolder, artifact.Format.ToString().ToLowerInvariant()); | ||||
|         return _fileSystem.Path.Combine(folder, sanitized + GetExtension(artifact.Format)); | ||||
|     } | ||||
|  | ||||
|     private void WriteOfflineBundle(string artifactRelativePath, VexExportArtifact artifact, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var zipPath = _fileSystem.Path.Combine(GetBundlesRoot(), artifact.ContentAddress.Digest.Replace(':', '_') + ".zip"); | ||||
|         using var zipStream = _fileSystem.File.Create(zipPath); | ||||
|         using var archive = new ZipArchive(zipStream, ZipArchiveMode.Create); | ||||
|         var entry = archive.CreateEntry(artifactRelativePath, CompressionLevel.Optimal); | ||||
|         using (var entryStream = entry.Open()) | ||||
|         { | ||||
|             entryStream.Write(artifact.Content.Span); | ||||
|         } | ||||
|  | ||||
|         // embed metadata file | ||||
|         var metadataEntry = archive.CreateEntry("metadata.json", CompressionLevel.Optimal); | ||||
|         using var metadataStream = new StreamWriter(metadataEntry.Open()); | ||||
|         var metadata = new Dictionary<string, object?> | ||||
|         { | ||||
|             ["digest"] = artifact.ContentAddress.ToUri(), | ||||
|             ["format"] = artifact.Format.ToString().ToLowerInvariant(), | ||||
|             ["sizeBytes"] = artifact.Content.Length, | ||||
|             ["metadata"] = artifact.Metadata, | ||||
|         }; | ||||
|         metadataStream.Write(JsonSerializer.Serialize(metadata, _serializerOptions)); | ||||
|     } | ||||
|  | ||||
|     private async Task UpdateManifestAsync(string artifactRelativePath, VexExportArtifact artifact, CancellationToken cancellationToken) | ||||
|     { | ||||
|         var manifestPath = _fileSystem.Path.Combine(_options.RootPath, _options.ManifestFileName); | ||||
|         var records = new List<ManifestEntry>(); | ||||
|  | ||||
|         if (_fileSystem.File.Exists(manifestPath)) | ||||
|         { | ||||
|             await using var existingStream = _fileSystem.File.OpenRead(manifestPath); | ||||
|             var existing = await JsonSerializer.DeserializeAsync<ManifestDocument>(existingStream, _serializerOptions, cancellationToken).ConfigureAwait(false); | ||||
|             if (existing is not null) | ||||
|             { | ||||
|                 records.AddRange(existing.Artifacts); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         records.RemoveAll(x => string.Equals(x.Digest, artifact.ContentAddress.ToUri(), StringComparison.OrdinalIgnoreCase)); | ||||
|         records.Add(new ManifestEntry( | ||||
|             artifact.ContentAddress.ToUri(), | ||||
|             artifact.Format.ToString().ToLowerInvariant(), | ||||
|             artifactRelativePath.Replace("\\", "/"), | ||||
|             artifact.Content.Length, | ||||
|             artifact.Metadata)); | ||||
|  | ||||
|         records.Sort(static (a, b) => string.CompareOrdinal(a.Digest, b.Digest)); | ||||
|  | ||||
|         var doc = new ManifestDocument(records.ToImmutableArray()); | ||||
|  | ||||
|         await using var stream = _fileSystem.File.Create(manifestPath); | ||||
|         await JsonSerializer.SerializeAsync(stream, doc, _serializerOptions, cancellationToken).ConfigureAwait(false); | ||||
|     } | ||||
|  | ||||
|     private string GetArtifactsRoot() => _fileSystem.Path.Combine(_options.RootPath, _options.ArtifactsFolder); | ||||
|  | ||||
|     private string GetBundlesRoot() => _fileSystem.Path.Combine(_options.RootPath, _options.BundlesFolder); | ||||
|  | ||||
|     private static string GetExtension(VexExportFormat format) | ||||
|         => format switch | ||||
|         { | ||||
|             VexExportFormat.Json => ".json", | ||||
|             VexExportFormat.JsonLines => ".jsonl", | ||||
|             VexExportFormat.OpenVex => ".json", | ||||
|             VexExportFormat.Csaf => ".json", | ||||
|             _ => ".bin", | ||||
|         }; | ||||
|  | ||||
|     private sealed record ManifestDocument(ImmutableArray<ManifestEntry> Artifacts); | ||||
|  | ||||
|     private sealed record ManifestEntry(string Digest, string Format, string Path, long SizeBytes, IReadOnlyDictionary<string, string> Metadata); | ||||
| } | ||||
|  | ||||
| public static class OfflineBundleArtifactStoreServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddVexOfflineBundleArtifactStore(this IServiceCollection services, Action<OfflineBundleArtifactStoreOptions>? configure = null) | ||||
|     { | ||||
|         if (configure is not null) | ||||
|         { | ||||
|             services.Configure(configure); | ||||
|         } | ||||
|  | ||||
|         services.AddSingleton<IVexArtifactStore, OfflineBundleArtifactStore>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
							
								
								
									
										3
									
								
								src/StellaOps.Vexer.Export/Properties/AssemblyInfo.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										3
									
								
								src/StellaOps.Vexer.Export/Properties/AssemblyInfo.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,3 @@ | ||||
| using System.Runtime.CompilerServices; | ||||
|  | ||||
| [assembly: InternalsVisibleTo("StellaOps.Vexer.Export.Tests")] | ||||
							
								
								
									
										181
									
								
								src/StellaOps.Vexer.Export/S3ArtifactStore.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										181
									
								
								src/StellaOps.Vexer.Export/S3ArtifactStore.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,181 @@ | ||||
| using System; | ||||
| using System.Collections.Generic; | ||||
| using System.IO; | ||||
| using System.Threading; | ||||
| using System.Threading.Tasks; | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using Microsoft.Extensions.Options; | ||||
| using StellaOps.Vexer.Core; | ||||
|  | ||||
| namespace StellaOps.Vexer.Export; | ||||
|  | ||||
| public sealed class S3ArtifactStoreOptions | ||||
| { | ||||
|     public string BucketName { get; set; } = string.Empty; | ||||
|  | ||||
|     public string? Prefix { get; set; } | ||||
|         = null; | ||||
|  | ||||
|     public bool OverwriteExisting { get; set; } | ||||
|         = true; | ||||
| } | ||||
|  | ||||
| public interface IS3ArtifactClient | ||||
| { | ||||
|     Task<bool> ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken); | ||||
|  | ||||
|     Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary<string, string> metadata, CancellationToken cancellationToken); | ||||
|  | ||||
|     Task<Stream?> GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken); | ||||
|  | ||||
|     Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken); | ||||
| } | ||||
|  | ||||
| public sealed class S3ArtifactStore : IVexArtifactStore | ||||
| { | ||||
|     private readonly IS3ArtifactClient _client; | ||||
|     private readonly S3ArtifactStoreOptions _options; | ||||
|     private readonly ILogger<S3ArtifactStore> _logger; | ||||
|  | ||||
|     public S3ArtifactStore( | ||||
|         IS3ArtifactClient client, | ||||
|         IOptions<S3ArtifactStoreOptions> options, | ||||
|         ILogger<S3ArtifactStore> logger) | ||||
|     { | ||||
|         _client = client ?? throw new ArgumentNullException(nameof(client)); | ||||
|         ArgumentNullException.ThrowIfNull(options); | ||||
|         _options = options.Value; | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|  | ||||
|         if (string.IsNullOrWhiteSpace(_options.BucketName)) | ||||
|         { | ||||
|             throw new ArgumentException("BucketName must be provided for S3ArtifactStore.", nameof(options)); | ||||
|         } | ||||
|     } | ||||
|  | ||||
|     public async ValueTask<VexStoredArtifact> SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(artifact); | ||||
|         var key = BuildObjectKey(artifact.ContentAddress, artifact.Format); | ||||
|  | ||||
|         if (!_options.OverwriteExisting) | ||||
|         { | ||||
|             var exists = await _client.ObjectExistsAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); | ||||
|             if (exists) | ||||
|             { | ||||
|                 _logger.LogInformation("S3 object {Bucket}/{Key} already exists; skipping upload.", _options.BucketName, key); | ||||
|                 return new VexStoredArtifact(artifact.ContentAddress, key, artifact.Content.Length, artifact.Metadata); | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         using var contentStream = new MemoryStream(artifact.Content.ToArray()); | ||||
|         await _client.PutObjectAsync( | ||||
|             _options.BucketName, | ||||
|             key, | ||||
|             contentStream, | ||||
|             BuildObjectMetadata(artifact), | ||||
|             cancellationToken).ConfigureAwait(false); | ||||
|  | ||||
|         _logger.LogInformation("Uploaded export artifact {Digest} to {Bucket}/{Key}", artifact.ContentAddress.ToUri(), _options.BucketName, key); | ||||
|  | ||||
|         return new VexStoredArtifact( | ||||
|             artifact.ContentAddress, | ||||
|             key, | ||||
|             artifact.Content.Length, | ||||
|             artifact.Metadata); | ||||
|     } | ||||
|  | ||||
|     public async ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(contentAddress); | ||||
|         foreach (var key in BuildCandidateKeys(contentAddress)) | ||||
|         { | ||||
|             await _client.DeleteObjectAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); | ||||
|         } | ||||
|         _logger.LogInformation("Deleted export artifact {Digest} from {Bucket}", contentAddress.ToUri(), _options.BucketName); | ||||
|     } | ||||
|  | ||||
|     public async ValueTask<Stream?> OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(contentAddress); | ||||
|         foreach (var key in BuildCandidateKeys(contentAddress)) | ||||
|         { | ||||
|             var stream = await _client.GetObjectAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); | ||||
|             if (stream is not null) | ||||
|             { | ||||
|                 return stream; | ||||
|             } | ||||
|         } | ||||
|  | ||||
|         return null; | ||||
|     } | ||||
|  | ||||
|     private string BuildObjectKey(VexContentAddress address, VexExportFormat format) | ||||
|     { | ||||
|         var sanitizedDigest = address.Digest.Replace(':', '_'); | ||||
|         var prefix = string.IsNullOrWhiteSpace(_options.Prefix) ? string.Empty : _options.Prefix.TrimEnd('/') + "/"; | ||||
|         var formatSegment = format.ToString().ToLowerInvariant(); | ||||
|         return $"{prefix}{formatSegment}/{sanitizedDigest}{GetExtension(format)}"; | ||||
|     } | ||||
|  | ||||
|     private IEnumerable<string> BuildCandidateKeys(VexContentAddress address) | ||||
|     { | ||||
|         foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) | ||||
|         { | ||||
|             yield return BuildObjectKey(address, format); | ||||
|         } | ||||
|  | ||||
|         if (!string.IsNullOrWhiteSpace(_options.Prefix)) | ||||
|         { | ||||
|             yield return $"{_options.Prefix.TrimEnd('/')}/{address.Digest.Replace(':', '_')}"; | ||||
|         } | ||||
|  | ||||
|         yield return address.Digest.Replace(':', '_'); | ||||
|     } | ||||
|  | ||||
|     private static IDictionary<string, string> BuildObjectMetadata(VexExportArtifact artifact) | ||||
|     { | ||||
|         var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) | ||||
|         { | ||||
|             ["vex-format"] = artifact.Format.ToString().ToLowerInvariant(), | ||||
|             ["vex-digest"] = artifact.ContentAddress.ToUri(), | ||||
|             ["content-type"] = artifact.Format switch | ||||
|             { | ||||
|                 VexExportFormat.Json => "application/json", | ||||
|                 VexExportFormat.JsonLines => "application/json", | ||||
|                 VexExportFormat.OpenVex => "application/vnd.openvex+json", | ||||
|                 VexExportFormat.Csaf => "application/json", | ||||
|                 _ => "application/octet-stream", | ||||
|             }, | ||||
|         }; | ||||
|  | ||||
|         foreach (var kvp in artifact.Metadata) | ||||
|         { | ||||
|             metadata[$"meta-{kvp.Key}"] = kvp.Value; | ||||
|         } | ||||
|  | ||||
|         return metadata; | ||||
|     } | ||||
|  | ||||
|     private static string GetExtension(VexExportFormat format) | ||||
|         => format switch | ||||
|         { | ||||
|             VexExportFormat.Json => ".json", | ||||
|             VexExportFormat.JsonLines => ".jsonl", | ||||
|             VexExportFormat.OpenVex => ".json", | ||||
|             VexExportFormat.Csaf => ".json", | ||||
|             _ => ".bin", | ||||
|         }; | ||||
| } | ||||
|  | ||||
| public static class S3ArtifactStoreServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddVexS3ArtifactStore(this IServiceCollection services, Action<S3ArtifactStoreOptions> configure) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(configure); | ||||
|         services.Configure(configure); | ||||
|         services.AddSingleton<IVexArtifactStore, S3ArtifactStore>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
| @@ -9,6 +9,7 @@ | ||||
|   <ItemGroup> | ||||
|     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" /> | ||||
|     <PackageReference Include="Microsoft.Extensions.Options" Version="8.0.0" /> | ||||
|     <PackageReference Include="System.IO.Abstractions" Version="20.0.28" /> | ||||
|   </ItemGroup> | ||||
|   <ItemGroup> | ||||
|     <ProjectReference Include="..\StellaOps.Vexer.Core\StellaOps.Vexer.Core.csproj" /> | ||||
|   | ||||
| @@ -3,6 +3,7 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and | ||||
| | Task | Owner(s) | Depends on | Notes | | ||||
| |---|---|---|---| | ||||
| |VEXER-EXPORT-01-001 – Export engine orchestration|Team Vexer Export|VEXER-CORE-01-003|DONE (2025-10-15) – Export engine scaffolding with cache lookup, data source hooks, and deterministic manifest emission.| | ||||
| |VEXER-EXPORT-01-002 – Cache index & eviction hooks|Team Vexer Export|VEXER-EXPORT-01-001, VEXER-STORAGE-01-003|TODO – Wire cache lookup/write path against `vex.cache` collection and add GC utilities for Worker to prune stale entries deterministically.| | ||||
| |VEXER-EXPORT-01-003 – Artifact store adapters|Team Vexer Export|VEXER-EXPORT-01-001|TODO – Provide pluggable storage adapters (filesystem, S3/MinIO) with offline bundle packaging and hash verification.| | ||||
| |VEXER-EXPORT-01-002 – Cache index & eviction hooks|Team Vexer Export|VEXER-EXPORT-01-001, VEXER-STORAGE-01-003|**DONE (2025-10-16)** – Export engine now invalidates cache entries on force refresh, cache services expose prune/invalidate APIs, and storage maintenance trims expired/dangling records with Mongo2Go coverage.| | ||||
| |VEXER-EXPORT-01-003 – Artifact store adapters|Team Vexer Export|VEXER-EXPORT-01-001|**DONE (2025-10-16)** – Implemented multi-store pipeline with filesystem, S3-compatible, and offline bundle adapters (hash verification + manifest/zip output) plus unit coverage and DI hooks.| | ||||
| |VEXER-EXPORT-01-004 – Attestation handoff integration|Team Vexer Export|VEXER-EXPORT-01-001, VEXER-ATTEST-01-001|TODO – Connect export engine to attestation client, persist Rekor metadata, and reuse cached attestations.| | ||||
| |VEXER-EXPORT-01-005 – Score & resolve envelope surfaces|Team Vexer Export|VEXER-EXPORT-01-004, VEXER-CORE-02-001|TODO – Emit consensus+score envelopes in export manifests, include policy/scoring digests, and update offline bundle/ORAS layouts to carry signed VEX responses.| | ||||
|   | ||||
							
								
								
									
										54
									
								
								src/StellaOps.Vexer.Export/VexExportCacheService.cs
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										54
									
								
								src/StellaOps.Vexer.Export/VexExportCacheService.cs
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,54 @@ | ||||
| using Microsoft.Extensions.DependencyInjection; | ||||
| using Microsoft.Extensions.Logging; | ||||
| using StellaOps.Vexer.Core; | ||||
| using StellaOps.Vexer.Storage.Mongo; | ||||
|  | ||||
| namespace StellaOps.Vexer.Export; | ||||
|  | ||||
| public interface IVexExportCacheService | ||||
| { | ||||
|     ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken); | ||||
|  | ||||
|     ValueTask<int> PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken); | ||||
|  | ||||
|     ValueTask<int> PruneDanglingAsync(CancellationToken cancellationToken); | ||||
| } | ||||
|  | ||||
| internal sealed class VexExportCacheService : IVexExportCacheService | ||||
| { | ||||
|     private readonly IVexCacheIndex _cacheIndex; | ||||
|     private readonly IVexCacheMaintenance _maintenance; | ||||
|     private readonly ILogger<VexExportCacheService> _logger; | ||||
|  | ||||
|     public VexExportCacheService( | ||||
|         IVexCacheIndex cacheIndex, | ||||
|         IVexCacheMaintenance maintenance, | ||||
|         ILogger<VexExportCacheService> logger) | ||||
|     { | ||||
|         _cacheIndex = cacheIndex ?? throw new ArgumentNullException(nameof(cacheIndex)); | ||||
|         _maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance)); | ||||
|         _logger = logger ?? throw new ArgumentNullException(nameof(logger)); | ||||
|     } | ||||
|  | ||||
|     public async ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) | ||||
|     { | ||||
|         ArgumentNullException.ThrowIfNull(signature); | ||||
|         await _cacheIndex.RemoveAsync(signature, format, cancellationToken).ConfigureAwait(false); | ||||
|         _logger.LogInformation("Invalidated export cache entry {Signature} ({Format})", signature.Value, format); | ||||
|     } | ||||
|  | ||||
|     public ValueTask<int> PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken) | ||||
|         => _maintenance.RemoveExpiredAsync(asOf, cancellationToken); | ||||
|  | ||||
|     public ValueTask<int> PruneDanglingAsync(CancellationToken cancellationToken) | ||||
|         => _maintenance.RemoveMissingManifestReferencesAsync(cancellationToken); | ||||
| } | ||||
|  | ||||
| public static class VexExportCacheServiceCollectionExtensions | ||||
| { | ||||
|     public static IServiceCollection AddVexExportCacheServices(this IServiceCollection services) | ||||
|     { | ||||
|         services.AddSingleton<IVexExportCacheService, VexExportCacheService>(); | ||||
|         return services; | ||||
|     } | ||||
| } | ||||
		Reference in New Issue
	
	Block a user