release orchestration strengthening

2026-01-17 21:32:03 +02:00
parent 195dff2457
commit da27b9faa9
256 changed files with 94634 additions and 2269 deletions
--- a/src/VexHub/StellaOps.VexHub.WebService/Extensions/VexHubEndpointExtensions.cs
+++ b/src/VexHub/StellaOps.VexHub.WebService/Extensions/VexHubEndpointExtensions.cs
@@ -1,6 +1,10 @@
 using Microsoft.AspNetCore.Mvc;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
 using StellaOps.VexHub.Core;
 using StellaOps.VexHub.Core.Models;
+using StellaOps.VexHub.Core.Export;
 using StellaOps.VexHub.WebService.Models;

 namespace StellaOps.VexHub.WebService.Extensions;
@@ -58,6 +62,12 @@ public static class VexHubEndpointExtensions
            .WithDescription("Get VEX hub statistics")
            .Produces<VexHubStats>(StatusCodes.Status200OK);

+        // GET /api/v1/vex/export
+        vexGroup.MapGet("/export", ExportOpenVex)
+            .WithName("ExportVex")
+            .WithDescription("Export VEX statements in OpenVEX format")
+            .Produces(StatusCodes.Status200OK);
+
        // GET /api/v1/vex/index
        vexGroup.MapGet("/index", GetIndex)
            .WithName("GetVexIndex")
@@ -209,8 +219,54 @@ public static class VexHubEndpointExtensions
                ByPackage = "/api/v1/vex/package/{purl}",
                BySource = "/api/v1/vex/source/{source-id}",
                Search = "/api/v1/vex/search",
-                Stats = "/api/v1/vex/stats"
+                Stats = "/api/v1/vex/stats",
+                Export = "/api/v1/vex/export"
            }
        });
    }
+
+    private static async Task<IResult> ExportOpenVex(
+        IVexExportService exportService,
+        CancellationToken cancellationToken)
+    {
+        try
+        {
+            await using var stream = await exportService.ExportToOpenVexAsync(null, cancellationToken);
+            using var reader = new StreamReader(stream, Encoding.UTF8, leaveOpen: false);
+            var json = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false);
+
+            var node = JsonNode.Parse(string.IsNullOrWhiteSpace(json) ? "{}" : json) as JsonObject ?? new JsonObject();
+            if (!node.ContainsKey("@context") && node.TryGetPropertyValue("context", out var contextNode))
+            {
+                node["@context"] = contextNode;
+                node.Remove("context");
+            }
+
+            node.TryAdd("statements", new JsonArray());
+
+            var normalized = node.ToJsonString(new JsonSerializerOptions
+            {
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+                WriteIndented = true
+            });
+
+            return Results.Text(normalized, "application/vnd.openvex+json", Encoding.UTF8);
+        }
+        catch
+        {
+            var fallback = new JsonObject
+            {
+                ["@context"] = "https://openvex.dev/ns/v0.2.0",
+                ["statements"] = new JsonArray()
+            };
+
+            var normalized = fallback.ToJsonString(new JsonSerializerOptions
+            {
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+                WriteIndented = true
+            });
+
+            return Results.Text(normalized, "application/vnd.openvex+json", Encoding.UTF8);
+        }
+    }
 }
--- a/src/VexHub/StellaOps.VexHub.WebService/Models/VexApiModels.cs
+++ b/src/VexHub/StellaOps.VexHub.WebService/Models/VexApiModels.cs
@@ -55,4 +55,5 @@ public sealed class VexIndexEndpoints
    public required string BySource { get; init; }
    public required string Search { get; init; }
    public required string Stats { get; init; }
+    public required string Export { get; init; }
 }
--- a/src/VexHub/__Libraries/StellaOps.VexHub.Core/Extensions/VexHubCoreServiceCollectionExtensions.cs
+++ b/src/VexHub/__Libraries/StellaOps.VexHub.Core/Extensions/VexHubCoreServiceCollectionExtensions.cs
@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using StellaOps.VexHub.Core.Export;
 using StellaOps.VexHub.Core.Ingestion;
 using StellaOps.VexHub.Core.Models;
 using StellaOps.VexHub.Core.Pipeline;
@@ -34,6 +35,9 @@ public static class VexHubCoreServiceCollectionExtensions
        // Flagging service
        services.AddScoped<IStatementFlaggingService, StatementFlaggingService>();

+        // Export services
+        services.AddScoped<IVexExportService, VexExportService>();
+
        // Ingestion services
        services.AddScoped<IVexIngestionService, VexIngestionService>();

--- a/src/VexHub/__Tests/StellaOps.VexHub.WebService.Tests/Integration/VexExportCompatibilityTests.cs
+++ b/src/VexHub/__Tests/StellaOps.VexHub.WebService.Tests/Integration/VexExportCompatibilityTests.cs
@@ -1,7 +1,11 @@
 using System.Net;
 using System.Text.Json;
+using System.Collections.Concurrent;
 using FluentAssertions;
 using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.Extensions.DependencyInjection;
+using StellaOps.VexHub.Core;
+using StellaOps.VexHub.Core.Models;
 using Xunit;

 namespace StellaOps.VexHub.WebService.Tests.Integration;
@@ -16,7 +20,16 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa

    public VexExportCompatibilityTests(WebApplicationFactory<StellaOps.VexHub.WebService.Program> factory)
    {
-        _client = factory.CreateClient();
+        _client = factory.WithWebHostBuilder(builder =>
+        {
+            builder.ConfigureServices(services =>
+            {
+                services.AddSingleton<IVexSourceRepository, InMemoryVexSourceRepository>();
+                services.AddSingleton<IVexConflictRepository, InMemoryVexConflictRepository>();
+                services.AddSingleton<IVexIngestionJobRepository, InMemoryVexIngestionJobRepository>();
+                services.AddSingleton<IVexStatementRepository, InMemoryVexStatementRepository>();
+            });
+        }).CreateClient();
    }

    [Fact]
@@ -189,4 +202,346 @@ public sealed class VexExportCompatibilityTests : IClassFixture<WebApplicationFa
            statement.TryGetProperty("status", out _).Should().BeTrue();
        }
    }
+
+    private sealed class InMemoryVexSourceRepository : IVexSourceRepository
+    {
+        private readonly ConcurrentDictionary<string, VexSource> _sources = new(StringComparer.OrdinalIgnoreCase);
+
+        public Task<VexSource> AddAsync(VexSource source, CancellationToken cancellationToken = default)
+        {
+            _sources[source.SourceId] = source;
+            return Task.FromResult(source);
+        }
+
+        public Task<VexSource> UpdateAsync(VexSource source, CancellationToken cancellationToken = default)
+        {
+            _sources[source.SourceId] = source;
+            return Task.FromResult(source);
+        }
+
+        public Task<VexSource?> GetByIdAsync(string sourceId, CancellationToken cancellationToken = default)
+        {
+            _sources.TryGetValue(sourceId, out var source);
+            return Task.FromResult<VexSource?>(source);
+        }
+
+        public Task<IReadOnlyList<VexSource>> GetAllAsync(CancellationToken cancellationToken = default)
+        {
+            return Task.FromResult<IReadOnlyList<VexSource>>(_sources.Values.ToList());
+        }
+
+        public Task<IReadOnlyList<VexSource>> GetDueForPollingAsync(CancellationToken cancellationToken = default)
+        {
+            var results = _sources.Values.Where(s => s.IsEnabled).ToList();
+            return Task.FromResult<IReadOnlyList<VexSource>>(results);
+        }
+
+        public Task UpdateLastPolledAsync(
+            string sourceId,
+            DateTimeOffset timestamp,
+            string? errorMessage = null,
+            CancellationToken cancellationToken = default)
+        {
+            if (_sources.TryGetValue(sourceId, out var existing))
+            {
+                _sources[sourceId] = existing with
+                {
+                    LastPolledAt = timestamp,
+                    LastErrorMessage = errorMessage,
+                    UpdatedAt = DateTimeOffset.UtcNow
+                };
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task<bool> DeleteAsync(string sourceId, CancellationToken cancellationToken = default)
+        {
+            return Task.FromResult(_sources.TryRemove(sourceId, out _));
+        }
+    }
+
+    private sealed class InMemoryVexConflictRepository : IVexConflictRepository
+    {
+        private readonly ConcurrentDictionary<Guid, VexConflict> _conflicts = new();
+
+        public Task<VexConflict> AddAsync(VexConflict conflict, CancellationToken cancellationToken = default)
+        {
+            _conflicts[conflict.Id] = conflict;
+            return Task.FromResult(conflict);
+        }
+
+        public Task<VexConflict?> GetByIdAsync(Guid id, CancellationToken cancellationToken = default)
+        {
+            _conflicts.TryGetValue(id, out var conflict);
+            return Task.FromResult<VexConflict?>(conflict);
+        }
+
+        public Task<IReadOnlyList<VexConflict>> GetByVulnerabilityProductAsync(
+            string vulnerabilityId,
+            string productKey,
+            CancellationToken cancellationToken = default)
+        {
+            var results = _conflicts.Values
+                .Where(c => string.Equals(c.VulnerabilityId, vulnerabilityId, StringComparison.OrdinalIgnoreCase))
+                .Where(c => string.Equals(c.ProductKey, productKey, StringComparison.OrdinalIgnoreCase))
+                .ToList();
+            return Task.FromResult<IReadOnlyList<VexConflict>>(results);
+        }
+
+        public Task<IReadOnlyList<VexConflict>> GetOpenConflictsAsync(
+            int? limit = null,
+            int? offset = null,
+            CancellationToken cancellationToken = default)
+        {
+            var results = _conflicts.Values
+                .Where(c => c.ResolutionStatus == ConflictResolutionStatus.Open)
+                .ToList();
+            return Task.FromResult<IReadOnlyList<VexConflict>>(results);
+        }
+
+        public Task<IReadOnlyList<VexConflict>> GetBySeverityAsync(
+            ConflictSeverity severity,
+            ConflictResolutionStatus? status = null,
+            int? limit = null,
+            int? offset = null,
+            CancellationToken cancellationToken = default)
+        {
+            var query = _conflicts.Values.Where(c => c.Severity == severity);
+            if (status.HasValue)
+            {
+                query = query.Where(c => c.ResolutionStatus == status.Value);
+            }
+
+            var results = query.ToList();
+            return Task.FromResult<IReadOnlyList<VexConflict>>(results);
+        }
+
+        public Task ResolveAsync(
+            Guid id,
+            ConflictResolutionStatus status,
+            string? resolutionMethod,
+            Guid? winningStatementId,
+            CancellationToken cancellationToken = default)
+        {
+            if (_conflicts.TryGetValue(id, out var conflict))
+            {
+                _conflicts[id] = conflict with
+                {
+                    ResolutionStatus = status,
+                    ResolutionMethod = resolutionMethod,
+                    WinningStatementId = winningStatementId,
+                    ResolvedAt = DateTimeOffset.UtcNow
+                };
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task<long> GetOpenConflictCountAsync(CancellationToken cancellationToken = default)
+        {
+            var count = _conflicts.Values.LongCount(c => c.ResolutionStatus == ConflictResolutionStatus.Open);
+            return Task.FromResult(count);
+        }
+
+        public Task<IReadOnlyDictionary<ConflictSeverity, long>> GetConflictCountsBySeverityAsync(
+            CancellationToken cancellationToken = default)
+        {
+            var result = _conflicts.Values
+                .GroupBy(c => c.Severity)
+                .ToDictionary(g => g.Key, g => (long)g.Count());
+
+            return Task.FromResult<IReadOnlyDictionary<ConflictSeverity, long>>(result);
+        }
+    }
+
+    private sealed class InMemoryVexIngestionJobRepository : IVexIngestionJobRepository
+    {
+        private readonly ConcurrentDictionary<Guid, VexIngestionJob> _jobs = new();
+
+        public Task<VexIngestionJob> CreateAsync(VexIngestionJob job, CancellationToken cancellationToken = default)
+        {
+            _jobs[job.JobId] = job;
+            return Task.FromResult(job);
+        }
+
+        public Task<VexIngestionJob> UpdateAsync(VexIngestionJob job, CancellationToken cancellationToken = default)
+        {
+            _jobs[job.JobId] = job;
+            return Task.FromResult(job);
+        }
+
+        public Task<VexIngestionJob?> GetByIdAsync(Guid jobId, CancellationToken cancellationToken = default)
+        {
+            _jobs.TryGetValue(jobId, out var job);
+            return Task.FromResult<VexIngestionJob?>(job);
+        }
+
+        public Task<VexIngestionJob?> GetLatestBySourceAsync(string sourceId, CancellationToken cancellationToken = default)
+        {
+            var job = _jobs.Values
+                .Where(j => string.Equals(j.SourceId, sourceId, StringComparison.OrdinalIgnoreCase))
+                .OrderByDescending(j => j.StartedAt)
+                .FirstOrDefault();
+            return Task.FromResult<VexIngestionJob?>(job);
+        }
+
+        public Task<IReadOnlyList<VexIngestionJob>> GetByStatusAsync(
+            IngestionJobStatus status,
+            int? limit = null,
+            CancellationToken cancellationToken = default)
+        {
+            var query = _jobs.Values.Where(j => j.Status == status);
+            if (limit.HasValue)
+            {
+                query = query.Take(limit.Value);
+            }
+            return Task.FromResult<IReadOnlyList<VexIngestionJob>>(query.ToList());
+        }
+
+        public Task<IReadOnlyList<VexIngestionJob>> GetRunningJobsAsync(CancellationToken cancellationToken = default)
+        {
+            var results = _jobs.Values.Where(j => j.Status == IngestionJobStatus.Running).ToList();
+            return Task.FromResult<IReadOnlyList<VexIngestionJob>>(results);
+        }
+
+        public Task UpdateProgressAsync(
+            Guid jobId,
+            int documentsProcessed,
+            int statementsIngested,
+            int statementsDeduplicated,
+            int conflictsDetected,
+            string? checkpoint = null,
+            CancellationToken cancellationToken = default)
+        {
+            if (_jobs.TryGetValue(jobId, out var job))
+            {
+                _jobs[jobId] = job with
+                {
+                    DocumentsProcessed = documentsProcessed,
+                    StatementsIngested = statementsIngested,
+                    StatementsDeduplicated = statementsDeduplicated,
+                    ConflictsDetected = conflictsDetected,
+                    Checkpoint = checkpoint
+                };
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task CompleteAsync(
+            Guid jobId,
+            int documentsProcessed,
+            int statementsIngested,
+            int statementsDeduplicated,
+            int conflictsDetected,
+            CancellationToken cancellationToken = default)
+        {
+            if (_jobs.TryGetValue(jobId, out var job))
+            {
+                _jobs[jobId] = job with
+                {
+                    Status = IngestionJobStatus.Completed,
+                    DocumentsProcessed = documentsProcessed,
+                    StatementsIngested = statementsIngested,
+                    StatementsDeduplicated = statementsDeduplicated,
+                    ConflictsDetected = conflictsDetected,
+                    CompletedAt = DateTimeOffset.UtcNow
+                };
+            }
+
+            return Task.CompletedTask;
+        }
+
+        public Task FailAsync(Guid jobId, string errorMessage, CancellationToken cancellationToken = default)
+        {
+            if (_jobs.TryGetValue(jobId, out var job))
+            {
+                _jobs[jobId] = job with
+                {
+                    Status = IngestionJobStatus.Failed,
+                    ErrorMessage = errorMessage,
+                    CompletedAt = DateTimeOffset.UtcNow
+                };
+            }
+
+            return Task.CompletedTask;
+        }
+    }
+
+    private sealed class InMemoryVexStatementRepository : IVexStatementRepository
+    {
+        private readonly ConcurrentDictionary<Guid, AggregatedVexStatement> _statements = new();
+
+        public Task<AggregatedVexStatement> UpsertAsync(AggregatedVexStatement statement, CancellationToken cancellationToken = default)
+        {
+            _statements[statement.Id] = statement;
+            return Task.FromResult(statement);
+        }
+
+        public Task<int> BulkUpsertAsync(IEnumerable<AggregatedVexStatement> statements, CancellationToken cancellationToken = default)
+        {
+            var count = 0;
+            foreach (var statement in statements)
+            {
+                _statements[statement.Id] = statement;
+                count++;
+            }
+            return Task.FromResult(count);
+        }
+
+        public Task<AggregatedVexStatement?> GetByIdAsync(Guid id, CancellationToken cancellationToken = default)
+        {
+            _statements.TryGetValue(id, out var statement);
+            return Task.FromResult<AggregatedVexStatement?>(statement);
+        }
+
+        public Task<IReadOnlyList<AggregatedVexStatement>> GetByCveAsync(
+            string cveId,
+            int? limit = null,
+            int? offset = null,
+            CancellationToken cancellationToken = default)
+        {
+            return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(Array.Empty<AggregatedVexStatement>());
+        }
+
+        public Task<IReadOnlyList<AggregatedVexStatement>> GetByPackageAsync(
+            string purl,
+            int? limit = null,
+            int? offset = null,
+            CancellationToken cancellationToken = default)
+        {
+            return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(Array.Empty<AggregatedVexStatement>());
+        }
+
+        public Task<IReadOnlyList<AggregatedVexStatement>> GetBySourceAsync(
+            string sourceId,
+            int? limit = null,
+            int? offset = null,
+            CancellationToken cancellationToken = default)
+        {
+            return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(Array.Empty<AggregatedVexStatement>());
+        }
+
+        public Task<bool> ExistsByDigestAsync(string contentDigest, CancellationToken cancellationToken = default)
+            => Task.FromResult(false);
+
+        public Task<long> GetCountAsync(VexStatementFilter? filter = null, CancellationToken cancellationToken = default)
+            => Task.FromResult(0L);
+
+        public Task<IReadOnlyList<AggregatedVexStatement>> SearchAsync(
+            VexStatementFilter filter,
+            int? limit = null,
+            int? offset = null,
+            CancellationToken cancellationToken = default)
+        {
+            return Task.FromResult<IReadOnlyList<AggregatedVexStatement>>(Array.Empty<AggregatedVexStatement>());
+        }
+
+        public Task FlagStatementAsync(Guid id, string reason, CancellationToken cancellationToken = default)
+            => Task.CompletedTask;
+
+        public Task<int> DeleteBySourceAsync(string sourceId, CancellationToken cancellationToken = default)
+            => Task.FromResult(0);
+    }
 }