up

2025-12-01 21:16:22 +02:00
parent c11d87d252
commit 909d9b6220
208 changed files with 860954 additions and 832 deletions
--- a/src/Excititor/__Tests/StellaOps.Excititor.Core.Tests/Observations/VexLinksetUpdatedEventFactoryTests.cs
+++ b/src/Excititor/__Tests/StellaOps.Excititor.Core.Tests/Observations/VexLinksetUpdatedEventFactoryTests.cs
@@ -32,6 +32,13 @@ public sealed class VexLinksetUpdatedEventFactoryTests
            linksetId: "link-123",
            vulnerabilityId: "CVE-2025-0001",
            productKey: "pkg:demo/app",
+            scope: new VexProductScope(
+                ProductKey: "pkg:demo/app",
+                Type: "package",
+                Version: "1.0.0",
+                Purl: "pkg:demo/app@1.0.0",
+                Cpe: null,
+                Identifiers: ImmutableArray.Create("pkg:demo/app@1.0.0")),
            observations,
            disagreements,
            now);
--- a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/PolicyEndpointsTests.cs
+++ b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/PolicyEndpointsTests.cs
@@ -0,0 +1,97 @@
+using System.Net.Http.Json;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+using StellaOps.Excititor.Core;
+using StellaOps.Excititor.Storage.Mongo;
+using StellaOps.Excititor.WebService.Contracts;
+
+namespace StellaOps.Excititor.WebService.Tests;
+
+public sealed class PolicyEndpointsTests
+{
+    [Fact]
+    public async Task VexLookup_ReturnsStatements_ForAdvisoryAndPurl()
+    {
+        var claims = CreateSampleClaims();
+
+        using var factory = new TestWebApplicationFactory(
+            configureServices: services =>
+            {
+                services.RemoveAll<IVexClaimStore>();
+                services.AddSingleton<IVexClaimStore>(new StubClaimStore(claims));
+                services.AddTestAuthentication();
+            });
+
+        using var client = factory.CreateClient(new() { AllowAutoRedirect = false });
+        client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "vex.read");
+        client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test");
+
+        var request = new PolicyVexLookupRequest
+        {
+            AdvisoryKeys = new[] { "CVE-2025-1234" },
+            Purls = new[] { "pkg:maven/org.example/app@1.2.3" },
+            Limit = 10
+        };
+
+        var response = await client.PostAsJsonAsync("/policy/v1/vex/lookup", request);
+
+        response.EnsureSuccessStatusCode();
+        var body = await response.Content.ReadFromJsonAsync<PolicyVexLookupResponse>();
+
+        Assert.NotNull(body);
+        Assert.Single(body!.Results);
+        var result = body.Results.First();
+        Assert.Equal("CVE-2025-1234", result.AdvisoryKey);
+        Assert.Single(result.Statements);
+        var statement = result.Statements.First();
+        Assert.Equal("pkg:maven/org.example/app@1.2.3", statement.Purl);
+        Assert.Equal("affected", statement.Status.ToLowerInvariant());
+    }
+
+    private static IReadOnlyList<VexClaim> CreateSampleClaims()
+    {
+        var now = DateTimeOffset.Parse("2025-12-01T12:00:00Z");
+        var product = new VexProduct(
+            key: "pkg:maven/org.example/app",
+            name: "Example App",
+            version: "1.2.3",
+            purl: "pkg:maven/org.example/app@1.2.3");
+
+        var document = new VexClaimDocument(
+            format: VexDocumentFormat.Csaf,
+            digest: "sha256:deadbeef",
+            sourceUri: new Uri("https://example.org/advisory.json"),
+            revision: "v1",
+            signature: null);
+
+        var claim = new VexClaim(
+            vulnerabilityId: "CVE-2025-1234",
+            providerId: "ghsa",
+            product: product,
+            status: VexClaimStatus.Affected,
+            document: document,
+            firstSeen: now.AddHours(-1),
+            lastSeen: now);
+
+        return new[] { claim };
+    }
+
+    private sealed class StubClaimStore : IVexClaimStore
+    {
+        private readonly IReadOnlyList<VexClaim> _claims;
+
+        public StubClaimStore(IReadOnlyList<VexClaim> claims)
+        {
+            _claims = claims;
+        }
+
+        public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
+            => ValueTask.CompletedTask;
+
+        public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
+            => ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(_claims.Where(c => c.VulnerabilityId == vulnerabilityId && c.Product.Key == productKey).ToList());
+
+        public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
+            => ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(_claims.Where(c => c.VulnerabilityId == vulnerabilityId).Take(limit).ToList());
+    }
+}
--- a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj
+++ b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj
@@ -41,5 +41,6 @@
    <Compile Include="GraphTooltipFactoryTests.cs" />
    <Compile Include="AttestationVerifyEndpointTests.cs" />
    <Compile Include="OpenApiDiscoveryEndpointTests.cs" />
+    <Compile Include="PolicyEndpointsTests.cs" />
  </ItemGroup>
 </Project>
--- a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/TestServiceOverrides.cs
+++ b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/TestServiceOverrides.cs
@@ -212,6 +212,81 @@ internal static class TestServiceOverrides
            _records.Add(record);
            return Task.CompletedTask;
        }
+
+        public Task<AirgapImportRecord?> FindByBundleIdAsync(
+            string tenantId,
+            string bundleId,
+            string? mirrorGeneration,
+            CancellationToken cancellationToken)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var matches = _records
+                .Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase)
+                            && string.Equals(r.BundleId, bundleId, StringComparison.OrdinalIgnoreCase));
+
+            if (!string.IsNullOrWhiteSpace(mirrorGeneration))
+            {
+                matches = matches.Where(r => string.Equals(r.MirrorGeneration, mirrorGeneration, StringComparison.OrdinalIgnoreCase));
+            }
+
+            var result = matches
+                .OrderByDescending(r => r.MirrorGeneration, StringComparer.OrdinalIgnoreCase)
+                .FirstOrDefault();
+
+            return Task.FromResult(result);
+        }
+
+        public Task<IReadOnlyList<AirgapImportRecord>> ListAsync(
+            string tenantId,
+            string? publisherFilter,
+            DateTimeOffset? importedAfter,
+            int limit,
+            int offset,
+            CancellationToken cancellationToken)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var query = _records
+                .Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase));
+
+            if (!string.IsNullOrWhiteSpace(publisherFilter))
+            {
+                query = query.Where(r => string.Equals(r.Publisher, publisherFilter, StringComparison.OrdinalIgnoreCase));
+            }
+
+            if (importedAfter.HasValue)
+            {
+                query = query.Where(r => r.ImportedAt > importedAfter.Value);
+            }
+
+            var result = query
+                .OrderByDescending(r => r.ImportedAt)
+                .Skip(Math.Max(0, offset))
+                .Take(Math.Clamp(limit, 1, 1000))
+                .ToList()
+                .AsReadOnly();
+
+            return Task.FromResult((IReadOnlyList<AirgapImportRecord>)result);
+        }
+
+        public Task<int> CountAsync(
+            string tenantId,
+            string? publisherFilter,
+            DateTimeOffset? importedAfter,
+            CancellationToken cancellationToken)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var count = _records
+                .Where(r => string.Equals(r.TenantId, tenantId, StringComparison.OrdinalIgnoreCase))
+                .Where(r => string.IsNullOrWhiteSpace(publisherFilter) ||
+                            string.Equals(r.Publisher, publisherFilter, StringComparison.OrdinalIgnoreCase))
+                .Where(r => !importedAfter.HasValue || r.ImportedAt > importedAfter.Value)
+                .Count();
+
+            return Task.FromResult(count);
+        }
    }

    private sealed class StubIngestOrchestrator : IVexIngestOrchestrator
--- a/src/Excititor/__Tests/StellaOps.Excititor.Worker.Tests/DefaultVexProviderRunnerIntegrationTests.cs
+++ b/src/Excititor/__Tests/StellaOps.Excititor.Worker.Tests/DefaultVexProviderRunnerIntegrationTests.cs
@@ -323,14 +323,17 @@ public sealed class DefaultVexProviderRunnerIntegrationTests : IAsyncLifetime
        }
    }

-    private sealed class NoopClaimStore : IVexClaimStore
-    {
-        public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
-            => ValueTask.CompletedTask;
-
-        public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
-            => ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
-    }
+    private sealed class NoopClaimStore : IVexClaimStore
+    {
+        public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
+            => ValueTask.CompletedTask;
+
+        public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
+            => ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
+
+        public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
+            => ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
+    }

    private sealed class NoopNormalizerRouter : IVexNormalizerRouter
    {
--- a/src/Excititor/__Tests/StellaOps.Excititor.Worker.Tests/DefaultVexProviderRunnerTests.cs
+++ b/src/Excititor/__Tests/StellaOps.Excititor.Worker.Tests/DefaultVexProviderRunnerTests.cs
@@ -484,14 +484,17 @@ public sealed class DefaultVexProviderRunnerTests
            => ValueTask.FromResult<VexRawDocument?>(null);
    }

-    private sealed class NoopClaimStore : IVexClaimStore
-    {
-        public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
-            => ValueTask.CompletedTask;
-
-        public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
-            => ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
-    }
+    private sealed class NoopClaimStore : IVexClaimStore
+    {
+        public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
+            => ValueTask.CompletedTask;
+
+        public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
+            => ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
+
+        public ValueTask<IReadOnlyCollection<VexClaim>> FindByVulnerabilityAsync(string vulnerabilityId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null)
+            => ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(Array.Empty<VexClaim>());
+    }

    private sealed class NoopProviderStore : IVexProviderStore
    {
--- a/src/Excititor/__Tests/StellaOps.Excititor.Worker.Tests/Orchestration/VexWorkerOrchestratorClientTests.cs
+++ b/src/Excititor/__Tests/StellaOps.Excititor.Worker.Tests/Orchestration/VexWorkerOrchestratorClientTests.cs
@@ -2,6 +2,10 @@ using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging.Abstractions;
@@ -18,11 +22,6 @@ public class VexWorkerOrchestratorClientTests
 {
    private readonly InMemoryConnectorStateRepository _stateRepository = new();
    private readonly FakeTimeProvider _timeProvider = new();
-    private readonly IOptions<VexWorkerOrchestratorOptions> _options = Microsoft.Extensions.Options.Options.Create(new VexWorkerOrchestratorOptions
-    {
-        Enabled = true,
-        DefaultTenant = "test-tenant"
-    });

    [Fact]
    public async Task StartJobAsync_CreatesJobContext()
@@ -61,6 +60,172 @@ public class VexWorkerOrchestratorClientTests
        Assert.NotNull(state.LastHeartbeatAt);
    }

+    [Fact]
+    public async Task StartJobAsync_UsesOrchestratorClaim_WhenAvailable()
+    {
+        var jobId = Guid.NewGuid();
+        var leaseId = Guid.NewGuid();
+
+        var handler = new StubHandler(request =>
+        {
+            if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
+            {
+                return JsonResponse(new
+                {
+                    jobId,
+                    leaseId,
+                    jobType = "exc-vex-ingest",
+                    payload = "{}",
+                    payloadDigest = "sha256:abc",
+                    attempt = 1,
+                    maxAttempts = 3,
+                    leaseUntil = "2025-12-01T12:00:00Z",
+                    idempotencyKey = "abc",
+                    correlationId = "corr-1",
+                    runId = (Guid?)null,
+                    projectId = (string?)null
+                });
+            }
+
+            return new HttpResponseMessage(HttpStatusCode.NotFound);
+        });
+
+        var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
+        var client = CreateClient(httpClient, opts => opts.BaseAddress = httpClient.BaseAddress);
+
+        var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
+
+        Assert.Equal(jobId, context.RunId);
+        Assert.Equal(jobId, context.OrchestratorJobId);
+        Assert.Equal(leaseId, context.OrchestratorLeaseId);
+        Assert.Contains(handler.Requests, r => r.RequestUri!.AbsolutePath.EndsWith("/claim"));
+    }
+
+    [Fact]
+    public async Task SendHeartbeatAsync_ExtendsLeaseViaOrchestrator()
+    {
+        var jobId = Guid.NewGuid();
+        var leaseId = Guid.NewGuid();
+        var leaseUntil = DateTimeOffset.Parse("2025-12-01T12:05:00Z");
+
+        var handler = new StubHandler(request =>
+        {
+            if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
+            {
+                return JsonResponse(new
+                {
+                    jobId,
+                    leaseId,
+                    jobType = "exc-vex-ingest",
+                    payload = "{}",
+                    payloadDigest = "sha256:abc",
+                    attempt = 1,
+                    maxAttempts = 3,
+                    leaseUntil = "2025-12-01T12:00:00Z",
+                    idempotencyKey = "abc",
+                    correlationId = "corr-1",
+                    runId = (Guid?)null,
+                    projectId = (string?)null
+                });
+            }
+
+            if (request.RequestUri!.AbsolutePath.Contains("/heartbeat"))
+            {
+                return JsonResponse(new
+                {
+                    jobId,
+                    leaseId,
+                    leaseUntil,
+                    acknowledged = true
+                });
+            }
+
+            return new HttpResponseMessage(HttpStatusCode.NotFound);
+        });
+
+        var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
+        var client = CreateClient(httpClient, opts =>
+        {
+            opts.BaseAddress = httpClient.BaseAddress;
+            opts.HeartbeatExtend = TimeSpan.FromSeconds(45);
+        });
+
+        var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
+
+        var heartbeat = new VexWorkerHeartbeat(
+            VexWorkerHeartbeatStatus.Running,
+            Progress: 10,
+            QueueDepth: null,
+            LastArtifactHash: "sha256:abc",
+            LastArtifactKind: "vex-document",
+            ErrorCode: null,
+            RetryAfterSeconds: null);
+
+        await client.SendHeartbeatAsync(context, heartbeat);
+
+        Assert.Equal(leaseUntil, context.LeaseExpiresAt);
+        Assert.Contains(handler.Requests, r => r.RequestUri!.AbsolutePath.Contains("/heartbeat"));
+    }
+
+    [Fact]
+    public async Task SendHeartbeatAsync_StoresThrottleCommand_On429()
+    {
+        var jobId = Guid.NewGuid();
+        var leaseId = Guid.NewGuid();
+
+        var handler = new StubHandler(request =>
+        {
+            if (request.RequestUri!.AbsolutePath.EndsWith("/claim"))
+            {
+                return JsonResponse(new
+                {
+                    jobId,
+                    leaseId,
+                    jobType = "exc-vex-ingest",
+                    payload = "{}",
+                    payloadDigest = "sha256:abc",
+                    attempt = 1,
+                    maxAttempts = 3,
+                    leaseUntil = "2025-12-01T12:00:00Z",
+                    idempotencyKey = "abc",
+                    correlationId = "corr-1",
+                    runId = (Guid?)null,
+                    projectId = (string?)null
+                });
+            }
+
+            if (request.RequestUri!.AbsolutePath.Contains("/heartbeat"))
+            {
+                var error = new { error = "rate_limited", message = "slow down", jobId, retryAfterSeconds = 15 };
+                return JsonResponse(error, HttpStatusCode.TooManyRequests);
+            }
+
+            return new HttpResponseMessage(HttpStatusCode.NotFound);
+        });
+
+        var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://orch.test/") };
+        var client = CreateClient(httpClient, opts => opts.BaseAddress = httpClient.BaseAddress);
+
+        var context = await client.StartJobAsync("tenant-a", "connector-001", "checkpoint-123");
+
+        var heartbeat = new VexWorkerHeartbeat(
+            VexWorkerHeartbeatStatus.Running,
+            Progress: 5,
+            QueueDepth: null,
+            LastArtifactHash: null,
+            LastArtifactKind: null,
+            ErrorCode: null,
+            RetryAfterSeconds: null);
+
+        await client.SendHeartbeatAsync(context, heartbeat);
+
+        var command = await client.GetPendingCommandAsync(context);
+
+        Assert.NotNull(command);
+        Assert.Equal(VexWorkerCommandKind.Throttle, command!.Kind);
+        Assert.Equal(15, command.Throttle?.CooldownSeconds);
+    }
+
    [Fact]
    public async Task RecordArtifactAsync_TracksArtifactHash()
    {
@@ -140,12 +305,25 @@ public class VexWorkerOrchestratorClientTests
        Assert.Equal(3, context.NextSequence());
    }

-    private VexWorkerOrchestratorClient CreateClient()
-        => new(
+    private VexWorkerOrchestratorClient CreateClient(
+        HttpClient? httpClient = null,
+        Action<VexWorkerOrchestratorOptions>? configure = null)
+    {
+        var opts = new VexWorkerOrchestratorOptions
+        {
+            Enabled = true,
+            DefaultTenant = "test-tenant"
+        };
+
+        configure?.Invoke(opts);
+
+        return new VexWorkerOrchestratorClient(
            _stateRepository,
            _timeProvider,
-            _options,
-            NullLogger<VexWorkerOrchestratorClient>.Instance);
+            Microsoft.Extensions.Options.Options.Create(opts),
+            NullLogger<VexWorkerOrchestratorClient>.Instance,
+            httpClient);
+    }

    private sealed class FakeTimeProvider : TimeProvider
    {
@@ -175,4 +353,31 @@ public class VexWorkerOrchestratorClientTests
        public ValueTask<IReadOnlyCollection<VexConnectorState>> ListAsync(CancellationToken cancellationToken, MongoDB.Driver.IClientSessionHandle? session = null)
            => ValueTask.FromResult<IReadOnlyCollection<VexConnectorState>>(_states.Values.ToList());
    }
+
+    private sealed class StubHandler : HttpMessageHandler
+    {
+        private readonly Func<HttpRequestMessage, HttpResponseMessage> _responder;
+
+        public StubHandler(Func<HttpRequestMessage, HttpResponseMessage> responder)
+        {
+            _responder = responder;
+        }
+
+        public List<HttpRequestMessage> Requests { get; } = new();
+
+        protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            Requests.Add(request);
+            return Task.FromResult(_responder(request));
+        }
+    }
+
+    private static HttpResponseMessage JsonResponse(object payload, HttpStatusCode status = HttpStatusCode.OK)
+    {
+        var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase });
+        return new HttpResponseMessage(status)
+        {
+            Content = new StringContent(json, Encoding.UTF8, "application/json")
+        };
+    }
 }