feat(scanner): Implement Deno analyzer and associated tests

- Added Deno analyzer with comprehensive metadata and evidence structure. - Created a detailed implementation plan for Sprint 130 focusing on Deno analyzer. - Introduced AdvisoryAiGuardrailOptions for managing guardrail configurations. - Developed GuardrailPhraseLoader for loading blocked phrases from JSON files. - Implemented tests for AdvisoryGuardrailOptions binding and phrase loading. - Enhanced telemetry for Advisory AI with metrics tracking. - Added VexObservationProjectionService for querying VEX observations. - Created extensive tests for VexObservationProjectionService functionality. - Introduced Ruby language analyzer with tests for simple and complex workspaces. - Added Ruby application fixtures for testing purposes.
2025-11-12 10:01:54 +02:00
parent 0e8655cbb1
commit babb81af52
75 changed files with 3346 additions and 187 deletions
--- a/src/Excititor/StellaOps.Excititor.WebService/Contracts/VexObservationContracts.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Contracts/VexObservationContracts.cs
@@ -0,0 +1,46 @@
+using System;
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace StellaOps.Excititor.WebService.Contracts;
+
+public sealed record VexObservationProjectionResponse(
+    [property: JsonPropertyName("vulnerabilityId")] string VulnerabilityId,
+    [property: JsonPropertyName("productKey")] string ProductKey,
+    [property: JsonPropertyName("generatedAt") ] DateTimeOffset GeneratedAt,
+    [property: JsonPropertyName("totalCount")] int TotalCount,
+    [property: JsonPropertyName("truncated")] bool Truncated,
+    [property: JsonPropertyName("statements")] IReadOnlyList<VexObservationStatementResponse> Statements);
+
+public sealed record VexObservationStatementResponse(
+    [property: JsonPropertyName("observationId")] string ObservationId,
+    [property: JsonPropertyName("providerId")] string ProviderId,
+    [property: JsonPropertyName("status")] string Status,
+    [property: JsonPropertyName("justification")] string? Justification,
+    [property: JsonPropertyName("detail")] string? Detail,
+    [property: JsonPropertyName("firstSeen")] DateTimeOffset FirstSeen,
+    [property: JsonPropertyName("lastSeen")] DateTimeOffset LastSeen,
+    [property: JsonPropertyName("scope")] VexObservationScopeResponse Scope,
+    [property: JsonPropertyName("anchors")] IReadOnlyList<string> Anchors,
+    [property: JsonPropertyName("document")] VexObservationDocumentResponse Document,
+    [property: JsonPropertyName("signature")] VexObservationSignatureResponse? Signature);
+
+public sealed record VexObservationScopeResponse(
+    [property: JsonPropertyName("key")] string Key,
+    [property: JsonPropertyName("name")] string? Name,
+    [property: JsonPropertyName("version")] string? Version,
+    [property: JsonPropertyName("purl")] string? Purl,
+    [property: JsonPropertyName("cpe")] string? Cpe,
+    [property: JsonPropertyName("componentIdentifiers")] IReadOnlyList<string> ComponentIdentifiers);
+
+public sealed record VexObservationDocumentResponse(
+    [property: JsonPropertyName("digest")] string Digest,
+    [property: JsonPropertyName("format")] string Format,
+    [property: JsonPropertyName("revision")] string? Revision,
+    [property: JsonPropertyName("sourceUri")] string SourceUri);
+
+public sealed record VexObservationSignatureResponse(
+    [property: JsonPropertyName("type")] string Type,
+    [property: JsonPropertyName("keyId")] string? KeyId,
+    [property: JsonPropertyName("issuer")] string? Issuer,
+    [property: JsonPropertyName("verifiedAt")] DateTimeOffset? VerifiedAtUtc);
--- a/src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Program.Helpers.cs
@@ -1,12 +1,17 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Globalization;
 using System.Linq;
 using System.Text;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Primitives;
 using MongoDB.Bson;
+using StellaOps.Excititor.Core;
 using StellaOps.Excititor.Core.Aoc;
 using StellaOps.Excititor.Storage.Mongo;
+using StellaOps.Excititor.WebService.Contracts;
+using StellaOps.Excititor.WebService.Services;
 public partial class Program
 {
    private const string TenantHeaderName = "X-Stella-Tenant";
@@ -127,4 +132,106 @@ public partial class Program
                ["primaryCode"] = exception.PrimaryErrorCode,
            });
    }
+
+    private static ImmutableHashSet<string> BuildStringFilterSet(StringValues values)
+    {
+        if (values.Count == 0)
+        {
+            return ImmutableHashSet<string>.Empty;
+        }
+
+        var builder = ImmutableHashSet.CreateBuilder<string>(StringComparer.OrdinalIgnoreCase);
+        foreach (var value in values)
+        {
+            if (!string.IsNullOrWhiteSpace(value))
+            {
+                builder.Add(value.Trim());
+            }
+        }
+
+        return builder.ToImmutable();
+    }
+
+    private static ImmutableHashSet<VexClaimStatus> BuildStatusFilter(StringValues values)
+    {
+        if (values.Count == 0)
+        {
+            return ImmutableHashSet<VexClaimStatus>.Empty;
+        }
+
+        var builder = ImmutableHashSet.CreateBuilder<VexClaimStatus>();
+        foreach (var value in values)
+        {
+            if (Enum.TryParse<VexClaimStatus>(value, ignoreCase: true, out var status))
+            {
+                builder.Add(status);
+            }
+        }
+
+        return builder.ToImmutable();
+    }
+
+    private static DateTimeOffset? ParseSinceTimestamp(StringValues values)
+    {
+        if (values.Count == 0)
+        {
+            return null;
+        }
+
+        var candidate = values[0];
+        return DateTimeOffset.TryParse(candidate, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
+            ? parsed
+            : null;
+    }
+
+    private static int ResolveLimit(StringValues values, int defaultValue, int min, int max)
+    {
+        if (values.Count == 0)
+        {
+            return defaultValue;
+        }
+
+        if (!int.TryParse(values[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed))
+        {
+            return defaultValue;
+        }
+
+        return Math.Clamp(parsed, min, max);
+    }
+
+    private static VexObservationStatementResponse ToResponse(VexObservationStatementProjection projection)
+    {
+        var scope = projection.Scope;
+        var document = projection.Document;
+        var signature = projection.Signature;
+
+        return new VexObservationStatementResponse(
+            projection.ObservationId,
+            projection.ProviderId,
+            projection.Status.ToString().ToLowerInvariant(),
+            projection.Justification?.ToString().ToLowerInvariant(),
+            projection.Detail,
+            projection.FirstSeen,
+            projection.LastSeen,
+            new VexObservationScopeResponse(
+                scope.Key,
+                scope.Name,
+                scope.Version,
+                scope.Purl,
+                scope.Cpe,
+                scope.ComponentIdentifiers),
+            projection.Anchors,
+            new VexObservationDocumentResponse(
+                document.Digest,
+                document.Format.ToString().ToLowerInvariant(),
+                document.Revision,
+                document.SourceUri.ToString()),
+            signature is null
+                ? null
+                : new VexObservationSignatureResponse(
+                    signature.Type,
+                    signature.KeyId,
+                    signature.Issuer,
+                    signature.VerifiedAt));
+    }
 }
--- a/src/Excititor/StellaOps.Excititor.WebService/Program.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Program.cs
@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Collections.Immutable;
@@ -5,7 +6,10 @@ using System.Globalization;
 using System.Text;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Primitives;
 using StellaOps.Excititor.Attestation.Verification;
 using StellaOps.Excititor.Attestation.Extensions;
 using StellaOps.Excititor.Attestation;
@@ -51,9 +55,11 @@ services.AddVexAttestation();
 services.Configure<VexAttestationClientOptions>(configuration.GetSection("Excititor:Attestation:Client"));
 services.Configure<VexAttestationVerificationOptions>(configuration.GetSection("Excititor:Attestation:Verification"));
 services.AddVexPolicy();
-services.AddRedHatCsafConnector();
-services.Configure<MirrorDistributionOptions>(configuration.GetSection(MirrorDistributionOptions.SectionName));
-services.AddSingleton<MirrorRateLimiter>();
+services.AddRedHatCsafConnector();
+services.Configure<MirrorDistributionOptions>(configuration.GetSection(MirrorDistributionOptions.SectionName));
+services.AddSingleton<MirrorRateLimiter>();
+services.TryAddSingleton(TimeProvider.System);
+services.AddSingleton<IVexObservationProjectionService, VexObservationProjectionService>();

 var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor");
 if (rekorSection.Exists())
@@ -434,6 +440,60 @@ app.MapGet("/vex/raw/{digest}/provenance", async (
    return Results.Json(response);
 });

+app.MapGet("/v1/vex/observations/{vulnerabilityId}/{productKey}", async (
+    HttpContext context,
+    string vulnerabilityId,
+    string productKey,
+    [FromServices] IVexObservationProjectionService projectionService,
+    [FromServices] IOptions<VexMongoStorageOptions> storageOptions,
+    CancellationToken cancellationToken) =>
+{
+    var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
+    if (scopeResult is not null)
+    {
+        return scopeResult;
+    }
+
+    if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
+    {
+        return tenantError;
+    }
+
+    if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
+    {
+        return ValidationProblem("vulnerabilityId and productKey are required.");
+    }
+
+    var providerFilter = BuildStringFilterSet(context.Request.Query["providerId"]);
+    var statusFilter = BuildStatusFilter(context.Request.Query["status"]);
+    var since = ParseSinceTimestamp(context.Request.Query["since"]);
+    var limit = ResolveLimit(context.Request.Query["limit"], defaultValue: 200, min: 1, max: 500);
+
+    var request = new VexObservationProjectionRequest(
+        tenant,
+        vulnerabilityId.Trim(),
+        productKey.Trim(),
+        providerFilter,
+        statusFilter,
+        since,
+        limit);
+
+    var result = await projectionService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
+    var statements = result.Statements
+        .Select(ToResponse)
+        .ToList();
+
+    var response = new VexObservationProjectionResponse(
+        request.VulnerabilityId,
+        request.ProductKey,
+        result.GeneratedAtUtc,
+        result.TotalCount,
+        result.Truncated,
+        statements);
+
+    return Results.Json(response);
+});
+
 app.MapPost("/aoc/verify", async (
    HttpContext context,
    VexAocVerifyRequest? request,
--- a/src/Excititor/StellaOps.Excititor.WebService/Services/VexObservationProjectionService.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Services/VexObservationProjectionService.cs
@@ -0,0 +1,161 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Globalization;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using StellaOps.Excititor.Core;
+
+namespace StellaOps.Excititor.WebService.Services;
+
+internal interface IVexObservationProjectionService
+{
+    Task<VexObservationProjectionResult> QueryAsync(
+        VexObservationProjectionRequest request,
+        CancellationToken cancellationToken);
+}
+
+internal sealed record VexObservationProjectionRequest(
+    string Tenant,
+    string VulnerabilityId,
+    string ProductKey,
+    ImmutableHashSet<string> ProviderIds,
+    ImmutableHashSet<VexClaimStatus> Statuses,
+    DateTimeOffset? Since,
+    int Limit);
+
+internal sealed record VexObservationProjectionResult(
+    IReadOnlyList<VexObservationStatementProjection> Statements,
+    bool Truncated,
+    int TotalCount,
+    DateTimeOffset GeneratedAtUtc);
+
+internal sealed record VexObservationStatementProjection(
+    string ObservationId,
+    string ProviderId,
+    VexClaimStatus Status,
+    VexJustification? Justification,
+    string? Detail,
+    DateTimeOffset FirstSeen,
+    DateTimeOffset LastSeen,
+    VexProductScope Scope,
+    IReadOnlyList<string> Anchors,
+    VexClaimDocument Document,
+    VexSignatureMetadata? Signature);
+
+internal sealed record VexProductScope(
+    string Key,
+    string? Name,
+    string? Version,
+    string? Purl,
+    string? Cpe,
+    IReadOnlyList<string> ComponentIdentifiers);
+
+internal sealed class VexObservationProjectionService : IVexObservationProjectionService
+{
+    private static readonly string[] AnchorKeys =
+    {
+        "json_pointer",
+        "jsonPointer",
+        "statement_locator",
+        "locator",
+        "paragraph",
+        "section",
+        "path"
+    };
+
+    private readonly IVexClaimStore _claimStore;
+    private readonly TimeProvider _timeProvider;
+
+    public VexObservationProjectionService(IVexClaimStore claimStore, TimeProvider? timeProvider = null)
+    {
+        _claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore));
+        _timeProvider = timeProvider ?? TimeProvider.System;
+    }
+
+    public async Task<VexObservationProjectionResult> QueryAsync(
+        VexObservationProjectionRequest request,
+        CancellationToken cancellationToken)
+    {
+        ArgumentNullException.ThrowIfNull(request);
+        cancellationToken.ThrowIfCancellationRequested();
+
+        var claims = await _claimStore.FindAsync(
+            request.VulnerabilityId,
+            request.ProductKey,
+            request.Since,
+            cancellationToken)
+            .ConfigureAwait(false);
+
+        var filtered = claims
+            .Where(claim => MatchesProvider(claim, request.ProviderIds))
+            .Where(claim => MatchesStatus(claim, request.Statuses))
+            .OrderByDescending(claim => claim.LastSeen)
+            .ThenBy(claim => claim.ProviderId, StringComparer.Ordinal)
+            .ToList();
+
+        var total = filtered.Count;
+        var page = filtered.Take(request.Limit).ToList();
+        var statements = page
+            .Select(claim => MapClaim(claim))
+            .ToList();
+
+        return new VexObservationProjectionResult(
+            statements,
+            total > request.Limit,
+            total,
+            _timeProvider.GetUtcNow());
+    }
+
+    private static bool MatchesProvider(VexClaim claim, ImmutableHashSet<string> providers)
+        => providers.Count == 0 || providers.Contains(claim.ProviderId, StringComparer.OrdinalIgnoreCase);
+
+    private static bool MatchesStatus(VexClaim claim, ImmutableHashSet<VexClaimStatus> statuses)
+        => statuses.Count == 0 || statuses.Contains(claim.Status);
+
+    private static VexObservationStatementProjection MapClaim(VexClaim claim)
+    {
+        var observationId = string.Create(CultureInfo.InvariantCulture, $"{claim.ProviderId}:{claim.Document.Digest}");
+        var anchors = ExtractAnchors(claim.AdditionalMetadata);
+        var scope = new VexProductScope(
+            claim.Product.Key,
+            claim.Product.Name,
+            claim.Product.Version,
+            claim.Product.Purl,
+            claim.Product.Cpe,
+            claim.Product.ComponentIdentifiers);
+
+        return new VexObservationStatementProjection(
+            observationId,
+            claim.ProviderId,
+            claim.Status,
+            claim.Justification,
+            claim.Detail,
+            claim.FirstSeen,
+            claim.LastSeen,
+            scope,
+            anchors,
+            claim.Document,
+            claim.Document.Signature);
+    }
+
+    private static IReadOnlyList<string> ExtractAnchors(ImmutableSortedDictionary<string, string> metadata)
+    {
+        if (metadata.Count == 0)
+        {
+            return Array.Empty<string>();
+        }
+
+        var anchors = new List<string>();
+        foreach (var key in AnchorKeys)
+        {
+            if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
+            {
+                anchors.Add(value.Trim());
+            }
+        }
+
+        return anchors.Count == 0 ? Array.Empty<string>() : anchors;
+    }
+}
--- a/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/VexObservationProjectionServiceTests.cs
+++ b/src/Excititor/__Tests/StellaOps.Excititor.WebService.Tests/VexObservationProjectionServiceTests.cs
@@ -0,0 +1,150 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using FluentAssertions;
+using MongoDB.Driver;
+using StellaOps.Excititor.Core;
+using StellaOps.Excititor.Storage.Mongo;
+using StellaOps.Excititor.WebService.Services;
+using Xunit;
+
+namespace StellaOps.Excititor.WebService.Tests;
+
+public sealed class VexObservationProjectionServiceTests
+{
+    [Fact]
+    public async Task QueryAsync_FiltersByProviderAndStatus()
+    {
+        var now = new DateTimeOffset(2025, 11, 10, 12, 0, 0, TimeSpan.Zero);
+        var claims = new[]
+        {
+            CreateClaim("provider-a", VexClaimStatus.Affected, now.AddHours(-6), now.AddHours(-5)),
+            CreateClaim("provider-b", VexClaimStatus.NotAffected, now.AddHours(-4), now.AddHours(-3))
+        };
+
+        var store = new FakeClaimStore(claims);
+        var service = new VexObservationProjectionService(store, new FixedTimeProvider(now));
+        var request = new VexObservationProjectionRequest(
+            Tenant: "tenant-a",
+            VulnerabilityId: "CVE-2025-0001",
+            ProductKey: "pkg:docker/demo",
+            ProviderIds: ImmutableHashSet.Create("provider-b"),
+            Statuses: ImmutableHashSet.Create(VexClaimStatus.NotAffected),
+            Since: null,
+            Limit: 10);
+
+        var result = await service.QueryAsync(request, CancellationToken.None);
+
+        result.Truncated.Should().BeFalse();
+        result.TotalCount.Should().Be(1);
+        result.GeneratedAtUtc.Should().Be(now);
+        var statement = result.Statements.Single();
+        statement.ProviderId.Should().Be("provider-b");
+        statement.Status.Should().Be(VexClaimStatus.NotAffected);
+        statement.Justification.Should().Be(VexJustification.ComponentNotPresent);
+        statement.Anchors.Should().ContainSingle().Which.Should().Be("/statements/0");
+        statement.Scope.ComponentIdentifiers.Should().Contain("demo:component");
+        statement.Document.Digest.Should().Contain("provider-b");
+    }
+
+    [Fact]
+    public async Task QueryAsync_TruncatesWhenLimitExceeded()
+    {
+        var now = DateTimeOffset.UtcNow;
+        var claims = Enumerable.Range(0, 3)
+            .Select(index => CreateClaim($"provider-{index}", VexClaimStatus.NotAffected, now.AddHours(-index - 2), now.AddHours(-index - 1)))
+            .ToArray();
+
+        var store = new FakeClaimStore(claims);
+        var service = new VexObservationProjectionService(store, new FixedTimeProvider(now));
+        var request = new VexObservationProjectionRequest(
+            Tenant: "tenant-a",
+            VulnerabilityId: "CVE-2025-0001",
+            ProductKey: "pkg:docker/demo",
+            ProviderIds: ImmutableHashSet<string>.Empty,
+            Statuses: ImmutableHashSet<VexClaimStatus>.Empty,
+            Since: null,
+            Limit: 2);
+
+        var result = await service.QueryAsync(request, CancellationToken.None);
+
+        result.Truncated.Should().BeTrue();
+        result.TotalCount.Should().Be(3);
+        result.Statements.Should().HaveCount(2);
+    }
+
+    private static VexClaim CreateClaim(string providerId, VexClaimStatus status, DateTimeOffset firstSeen, DateTimeOffset lastSeen)
+    {
+        var product = new VexProduct(
+            key: "pkg:docker/demo",
+            name: "demo",
+            version: "1.0.0",
+            purl: "pkg:docker/demo@1.0.0",
+            cpe: "cpe:/a:demo:demo:1.0.0",
+            componentIdentifiers: new[] { "demo:component" });
+
+        var document = new VexClaimDocument(
+            VexDocumentFormat.Csaf,
+            $"sha256:{providerId}",
+            new Uri("https://example.org/vex.json"),
+            revision: "v1");
+
+        var metadata = ImmutableDictionary<string, string>.Empty.Add("json_pointer", "/statements/0");
+
+        return new VexClaim(
+            "CVE-2025-0001",
+            providerId,
+            product,
+            status,
+            document,
+            firstSeen,
+            lastSeen,
+            justification: VexJustification.ComponentNotPresent,
+            detail: "not affected",
+            confidence: null,
+            signals: null,
+            additionalMetadata: metadata);
+    }
+
+    private sealed class FakeClaimStore : IVexClaimStore
+    {
+        private readonly IReadOnlyCollection<VexClaim> _claims;
+
+        public FakeClaimStore(IReadOnlyCollection<VexClaim> claims)
+        {
+            _claims = claims;
+        }
+
+        public ValueTask AppendAsync(IEnumerable<VexClaim> claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null)
+            => throw new NotSupportedException();
+
+        public ValueTask<IReadOnlyCollection<VexClaim>> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null)
+        {
+            var query = _claims
+                .Where(claim => string.Equals(claim.VulnerabilityId, vulnerabilityId, StringComparison.OrdinalIgnoreCase))
+                .Where(claim => string.Equals(claim.Product.Key, productKey, StringComparison.OrdinalIgnoreCase));
+
+            if (since.HasValue)
+            {
+                query = query.Where(claim => claim.LastSeen >= since.Value);
+            }
+
+            return ValueTask.FromResult<IReadOnlyCollection<VexClaim>>(query.ToList());
+        }
+    }
+
+    private sealed class FixedTimeProvider : TimeProvider
+    {
+        private readonly DateTimeOffset _timestamp;
+
+        public FixedTimeProvider(DateTimeOffset timestamp)
+        {
+            _timestamp = timestamp;
+        }
+
+        public override DateTimeOffset GetUtcNow() => _timestamp;
+    }
+}