blockers 2

2025-11-23 14:54:17 +02:00
parent f47d2d1377
commit cce96f3596
100 changed files with 2758 additions and 1912 deletions
--- a/src/Excititor/StellaOps.Excititor.WebService/Contracts/GraphLinkoutsContracts.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Contracts/GraphLinkoutsContracts.cs
@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.Text.Json.Serialization;
+
+namespace StellaOps.Excititor.WebService.Contracts;
+
+public sealed record GraphLinkoutsRequest(
+    [property: JsonPropertyName("tenant")] string Tenant,
+    [property: JsonPropertyName("purls")] IReadOnlyList<string> Purls,
+    [property: JsonPropertyName("includeJustifications")] bool IncludeJustifications = false,
+    [property: JsonPropertyName("includeProvenance")] bool IncludeProvenance = true);
+
+public sealed record GraphLinkoutsResponse(
+    [property: JsonPropertyName("items")] IReadOnlyList<GraphLinkoutItem> Items,
+    [property: JsonPropertyName("notFound")] IReadOnlyList<string> NotFound);
+
+public sealed record GraphLinkoutItem(
+    [property: JsonPropertyName("purl")] string Purl,
+    [property: JsonPropertyName("advisories")] IReadOnlyList<GraphLinkoutAdvisory> Advisories,
+    [property: JsonPropertyName("conflicts")] IReadOnlyList<GraphLinkoutConflict> Conflicts,
+    [property: JsonPropertyName("truncated")] bool Truncated = false,
+    [property: JsonPropertyName("nextCursor")] string? NextCursor = null);
+
+public sealed record GraphLinkoutAdvisory(
+    [property: JsonPropertyName("advisoryId")] string AdvisoryId,
+    [property: JsonPropertyName("source")] string Source,
+    [property: JsonPropertyName("status")] string Status,
+    [property: JsonPropertyName("justification")] string? Justification,
+    [property: JsonPropertyName("modifiedAt")] DateTimeOffset ModifiedAt,
+    [property: JsonPropertyName("evidenceHash")] string EvidenceHash,
+    [property: JsonPropertyName("connectorId")] string ConnectorId,
+    [property: JsonPropertyName("dsseEnvelopeHash")] string? DsseEnvelopeHash);
+
+public sealed record GraphLinkoutConflict(
+    [property: JsonPropertyName("source")] string Source,
+    [property: JsonPropertyName("status")] string Status,
+    [property: JsonPropertyName("justification")] string? Justification,
+    [property: JsonPropertyName("observedAt")] DateTimeOffset ObservedAt,
+    [property: JsonPropertyName("evidenceHash")] string EvidenceHash);
+
--- a/src/Excititor/StellaOps.Excititor.WebService/Contracts/VexConsoleContracts.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Contracts/VexConsoleContracts.cs
@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+
+namespace StellaOps.Excititor.WebService.Contracts;
+
+public sealed record VexConsoleStatementDto(
+    string AdvisoryId,
+    string ProductKey,
+    string? Purl,
+    string Status,
+    string? Justification,
+    string ProviderId,
+    string ObservationId,
+    DateTimeOffset CreatedAtUtc,
+    IReadOnlyDictionary<string, string> Attributes);
+
+public sealed record VexConsolePage(
+    IReadOnlyList<VexConsoleStatementDto> Items,
+    string? Cursor,
+    bool HasMore,
+    int Returned,
+    IReadOnlyDictionary<string, int>? Counters = null);
--- a/src/Excititor/StellaOps.Excititor.WebService/Program.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Program.cs
@@ -36,6 +36,8 @@ using StellaOps.Excititor.WebService.Contracts;
 using StellaOps.Excititor.WebService.Telemetry;
 using MongoDB.Driver;
 using MongoDB.Bson;
+using Microsoft.Extensions.Caching.Memory;
+using StellaOps.Excititor.WebService.Contracts;

 var builder = WebApplication.CreateBuilder(args);
 var configuration = builder.Configuration;
@@ -49,7 +51,11 @@ services.AddCsafNormalizer();
 services.AddCycloneDxNormalizer();
 services.AddOpenVexNormalizer();
 services.AddSingleton<IVexSignatureVerifier, NoopVexSignatureVerifier>();
+// TODO: replace NoopVexSignatureVerifier with hardened verifier once portable bundle signatures are finalized.
 services.AddSingleton<AirgapImportValidator>();
+services.AddSingleton<AirgapSignerTrustService>();
+services.AddSingleton<ConsoleTelemetry>();
+services.AddMemoryCache();
 services.AddScoped<IVexIngestOrchestrator, VexIngestOrchestrator>();
 services.AddOptions<ExcititorObservabilityOptions>()
    .Bind(configuration.GetSection("Excititor:Observability"));
@@ -68,6 +74,7 @@ services.Configure<MirrorDistributionOptions>(configuration.GetSection(MirrorDis
 services.AddSingleton<MirrorRateLimiter>();
 services.TryAddSingleton(TimeProvider.System);
 services.AddSingleton<IVexObservationProjectionService, VexObservationProjectionService>();
+services.AddScoped<IVexObservationQueryService, VexObservationQueryService>();

 var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor");
 if (rekorSection.Exists())
@@ -140,6 +147,7 @@ app.MapHealthChecks("/excititor/health");

 app.MapPost("/airgap/v1/vex/import", async (
    [FromServices] AirgapImportValidator validator,
+    [FromServices] AirgapSignerTrustService trustService,
    [FromServices] IAirgapImportStore store,
    [FromServices] TimeProvider timeProvider,
    [FromBody] AirgapImportRequest request,
@@ -160,6 +168,18 @@ app.MapPost("/airgap/v1/vex/import", async (
        });
    }

+    if (!trustService.Validate(request, out var trustCode, out var trustMessage))
+    {
+        return Results.StatusCode(StatusCodes.Status403Forbidden, new
+        {
+            error = new
+            {
+                code = trustCode,
+                message = trustMessage
+            }
+        });
+    }
+
    var record = new AirgapImportRecord
    {
        Id = $"{request.BundleId}:{request.MirrorGeneration}",
@@ -174,7 +194,21 @@ app.MapPost("/airgap/v1/vex/import", async (
        ImportedAt = nowUtc
    };

-    await store.SaveAsync(record, cancellationToken).ConfigureAwait(false);
+    try
+    {
+        await store.SaveAsync(record, cancellationToken).ConfigureAwait(false);
+    }
+    catch (DuplicateAirgapImportException dup)
+    {
+        return Results.Conflict(new
+        {
+            error = new
+            {
+                code = "AIRGAP_IMPORT_DUPLICATE",
+                message = dup.Message
+            }
+        });
+    }

    return Results.Accepted($"/airgap/v1/vex/import/{request.BundleId}", new
    {
@@ -296,6 +330,204 @@ app.MapPost("/excititor/admin/backfill-statements", async (
    });
 });

+app.MapGet("/console/vex", async (
+    HttpContext context,
+    IOptions<VexMongoStorageOptions> storageOptions,
+    IVexObservationQueryService queryService,
+    ConsoleTelemetry telemetry,
+    IMemoryCache cache,
+    CancellationToken cancellationToken) =>
+{
+    if (!TryResolveTenant(context, storageOptions.Value, requireHeader: true, out var tenant, out var tenantError))
+    {
+        return tenantError;
+    }
+
+    var query = context.Request.Query;
+    var purls = query["purl"].Where(static v => !string.IsNullOrWhiteSpace(v)).Select(static v => v.Trim()).ToArray();
+    var advisories = query["advisoryId"].Where(static v => !string.IsNullOrWhiteSpace(v)).Select(static v => v.Trim()).ToArray();
+    var statuses = new List<VexClaimStatus>();
+    if (query.TryGetValue("status", out var statusValues))
+    {
+        foreach (var statusValue in statusValues)
+        {
+            if (Enum.TryParse<VexClaimStatus>(statusValue, ignoreCase: true, out var parsed))
+            {
+                statuses.Add(parsed);
+            }
+            else
+            {
+                return Results.BadRequest($"Unknown status '{statusValue}'.");
+            }
+        }
+    }
+
+    var limit = query.TryGetValue("pageSize", out var pageSizeValues) && int.TryParse(pageSizeValues.FirstOrDefault(), out var pageSize)
+        ? pageSize
+        : (int?)null;
+    var cursor = query.TryGetValue("cursor", out var cursorValues) ? cursorValues.FirstOrDefault() : null;
+
+    telemetry.Requests.Add(1);
+
+    var cacheKey = $"console-vex:{tenant}:{string.Join(',', purls)}:{string.Join(',', advisories)}:{string.Join(',', statuses)}:{limit}:{cursor}";
+    if (cache.TryGetValue(cacheKey, out VexConsolePage? cachedPage) && cachedPage is not null)
+    {
+        telemetry.CacheHits.Add(1);
+        return Results.Ok(cachedPage);
+    }
+    telemetry.CacheMisses.Add(1);
+
+    var options = new VexObservationQueryOptions(
+        tenant,
+        observationIds: null,
+        vulnerabilityIds: advisories,
+        productKeys: null,
+        purls: purls,
+        cpes: null,
+        providerIds: null,
+        statuses: statuses,
+        cursor: cursor,
+        limit: limit);
+
+    VexObservationQueryResult result;
+    try
+    {
+        result = await queryService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
+    }
+    catch (FormatException ex)
+    {
+        return Results.BadRequest(ex.Message);
+    }
+
+    var statements = result.Observations
+        .SelectMany(obs => obs.Statements.Select(stmt => new VexConsoleStatementDto(
+            AdvisoryId: stmt.VulnerabilityId,
+            ProductKey: stmt.ProductKey,
+            Purl: stmt.Purl ?? obs.Linkset.Purls.FirstOrDefault(),
+            Status: stmt.Status.ToString().ToLowerInvariant(),
+            Justification: stmt.Justification?.ToString(),
+            ProviderId: obs.ProviderId,
+            ObservationId: obs.ObservationId,
+            CreatedAtUtc: obs.CreatedAt,
+            Attributes: obs.Attributes)))
+        .ToList();
+
+    var statusCounts = result.Observations
+        .GroupBy(o => o.Status.ToString().ToLowerInvariant())
+        .ToDictionary(g => g.Key, g => g.Count(), StringComparer.OrdinalIgnoreCase);
+
+    var response = new VexConsolePage(
+        Items: statements,
+        Cursor: result.NextCursor,
+        HasMore: result.HasMore,
+        Returned: statements.Count,
+        Counters: statusCounts);
+
+    cache.Set(cacheKey, response, TimeSpan.FromSeconds(30));
+
+    return Results.Ok(response);
+}).WithName("GetConsoleVex");
+
+// Cartographer linkouts
+app.MapPost("/internal/graph/linkouts", async (
+    GraphLinkoutsRequest request,
+    IVexObservationQueryService queryService,
+    CancellationToken cancellationToken) =>
+{
+    if (request is null || string.IsNullOrWhiteSpace(request.Tenant))
+    {
+        return Results.BadRequest("tenant is required.");
+    }
+
+    if (request.Purls is null || request.Purls.Count == 0 || request.Purls.Count > 500)
+    {
+        return Results.BadRequest("purls are required (1-500).");
+    }
+
+    var normalizedPurls = request.Purls
+        .Where(p => !string.IsNullOrWhiteSpace(p))
+        .Select(p => p.Trim().ToLowerInvariant())
+        .Distinct()
+        .ToArray();
+
+    if (normalizedPurls.Length == 0)
+    {
+        return Results.BadRequest("purls are required (1-500).");
+    }
+
+    var options = new VexObservationQueryOptions(
+        request.Tenant.Trim(),
+        purls: normalizedPurls,
+        includeJustifications: request.IncludeJustifications,
+        includeProvenance: request.IncludeProvenance,
+        limit: 200);
+
+    VexObservationQueryResult result;
+    try
+    {
+        result = await queryService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
+    }
+    catch (FormatException ex)
+    {
+        return Results.BadRequest(ex.Message);
+    }
+
+    var observationsByPurl = result.Observations
+        .SelectMany(obs => obs.Linkset.Purls.Select(purl => (purl, obs)))
+        .GroupBy(tuple => tuple.purl, StringComparer.OrdinalIgnoreCase)
+        .ToDictionary(g => g.Key, g => g.Select(t => t.obs).ToArray(), StringComparer.OrdinalIgnoreCase);
+
+    var items = new List<GraphLinkoutItem>(normalizedPurls.Length);
+    var notFound = new List<string>();
+
+    foreach (var inputPurl in normalizedPurls)
+    {
+        if (!observationsByPurl.TryGetValue(inputPurl, out var obsForPurl))
+        {
+            notFound.Add(inputPurl);
+            continue;
+        }
+
+        var advisories = obsForPurl
+            .SelectMany(obs => obs.Statements.Select(stmt => new GraphLinkoutAdvisory(
+                AdvisoryId: stmt.VulnerabilityId,
+                Source: obs.ProviderId,
+                Status: stmt.Status.ToString().ToLowerInvariant(),
+                Justification: request.IncludeJustifications ? stmt.Justification?.ToString() : null,
+                ModifiedAt: obs.CreatedAt,
+                EvidenceHash: obs.Linkset.ReferenceHash,
+                ConnectorId: obs.ProviderId,
+                DsseEnvelopeHash: request.IncludeProvenance ? obs.Linkset.ReferenceHash : null)))
+            .OrderBy(a => a.AdvisoryId, StringComparer.Ordinal)
+            .ThenBy(a => a.Source, StringComparer.Ordinal)
+            .Take(200)
+            .ToList();
+
+        var conflicts = obsForPurl
+            .Where(obs => obs.Statements.Any(s => s.Status == VexClaimStatus.Conflict))
+            .SelectMany(obs => obs.Statements
+                .Where(s => s.Status == VexClaimStatus.Conflict)
+                .Select(stmt => new GraphLinkoutConflict(
+                    Source: obs.ProviderId,
+                    Status: stmt.Status.ToString().ToLowerInvariant(),
+                    Justification: request.IncludeJustifications ? stmt.Justification?.ToString() : null,
+                    ObservedAt: obs.CreatedAt,
+                    EvidenceHash: obs.Linkset.ReferenceHash)))
+            .OrderBy(c => c.Source, StringComparer.Ordinal)
+            .ToList();
+
+        items.Add(new GraphLinkoutItem(
+            Purl: inputPurl,
+            Advisories: advisories,
+            Conflicts: conflicts,
+            Truncated: advisories.Count >= 200,
+            NextCursor: advisories.Count >= 200 ? $"{advisories[^1].AdvisoryId}:{advisories[^1].Source}" : null));
+    }
+
+    var response = new GraphLinkoutsResponse(items, notFound);
+    return Results.Ok(response);
+}).WithName("PostGraphLinkouts");
+
 app.MapPost("/ingest/vex", async (
    HttpContext context,
    VexIngestRequest request,
--- a/src/Excititor/StellaOps.Excititor.WebService/Properties/AssemblyInfo.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Properties/AssemblyInfo.cs
@@ -1,3 +1,4 @@
 using System.Runtime.CompilerServices;

-[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")]
+[assembly: InternalsVisibleTo("StellaOps.Excititor.WebService.Tests")]
+[assembly: InternalsVisibleTo("StellaOps.Excititor.Core.UnitTests")]
--- a/src/Excititor/StellaOps.Excititor.WebService/Services/AirgapImportValidator.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Services/AirgapImportValidator.cs
@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
+using System.Text.RegularExpressions;
 using StellaOps.Excititor.WebService.Contracts;

 namespace StellaOps.Excititor.WebService.Services;
@@ -8,6 +10,8 @@ namespace StellaOps.Excititor.WebService.Services;
 internal sealed class AirgapImportValidator
 {
    private static readonly TimeSpan AllowedSkew = TimeSpan.FromSeconds(5);
+    private static readonly Regex Sha256Pattern = new(@"^sha256:[A-Fa-f0-9]{64}$", RegexOptions.Compiled);
+    private static readonly Regex MirrorGenerationPattern = new(@"^[0-9]+$", RegexOptions.Compiled);

    public IReadOnlyList<ValidationError> Validate(AirgapImportRequest request, DateTimeOffset nowUtc)
    {
@@ -23,26 +27,46 @@ internal sealed class AirgapImportValidator
        {
            errors.Add(new ValidationError("bundle_id_missing", "bundleId is required."));
        }
+        else if (request.BundleId.Length > 256)
+        {
+            errors.Add(new ValidationError("bundle_id_too_long", "bundleId must be <= 256 characters."));
+        }

        if (string.IsNullOrWhiteSpace(request.MirrorGeneration))
        {
            errors.Add(new ValidationError("mirror_generation_missing", "mirrorGeneration is required."));
        }
+        else if (!MirrorGenerationPattern.IsMatch(request.MirrorGeneration))
+        {
+            errors.Add(new ValidationError("mirror_generation_invalid", "mirrorGeneration must be a numeric string."));
+        }

        if (string.IsNullOrWhiteSpace(request.Publisher))
        {
            errors.Add(new ValidationError("publisher_missing", "publisher is required."));
        }
+        else if (request.Publisher.Length > 256)
+        {
+            errors.Add(new ValidationError("publisher_too_long", "publisher must be <= 256 characters."));
+        }

        if (string.IsNullOrWhiteSpace(request.PayloadHash))
        {
            errors.Add(new ValidationError("payload_hash_missing", "payloadHash is required."));
        }
+        else if (!Sha256Pattern.IsMatch(request.PayloadHash))
+        {
+            errors.Add(new ValidationError("payload_hash_invalid", "payloadHash must be sha256:<64-hex>."));
+        }

        if (string.IsNullOrWhiteSpace(request.Signature))
        {
            errors.Add(new ValidationError("AIRGAP_SIGNATURE_MISSING", "signature is required for air-gapped imports."));
        }
+        else if (!IsBase64(request.Signature))
+        {
+            errors.Add(new ValidationError("AIRGAP_SIGNATURE_INVALID", "signature must be base64-encoded."));
+        }

        if (request.SignedAt is null)
        {
@@ -62,5 +86,22 @@ internal sealed class AirgapImportValidator
        return errors;
    }

+    private static bool IsBase64(string value)
+    {
+        if (string.IsNullOrWhiteSpace(value) || value.Length % 4 != 0)
+        {
+            return false;
+        }
+        try
+        {
+            _ = Convert.FromBase64String(value);
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
    public readonly record struct ValidationError(string Code, string Message);
 }
--- a/src/Excititor/StellaOps.Excititor.WebService/Services/AirgapSignerTrustService.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Services/AirgapSignerTrustService.cs
@@ -0,0 +1,82 @@
+using System;
+using System.IO;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using StellaOps.Excititor.Connectors.Abstractions.Trust;
+using StellaOps.Excititor.WebService.Contracts;
+
+namespace StellaOps.Excititor.WebService.Services;
+
+internal sealed class AirgapSignerTrustService
+{
+    private readonly ILogger<AirgapSignerTrustService> _logger;
+    private readonly string? _metadataPath;
+    private ConnectorSignerMetadataSet? _metadata;
+
+    public AirgapSignerTrustService(ILogger<AirgapSignerTrustService> logger)
+    {
+        _logger = logger;
+        _metadataPath = Environment.GetEnvironmentVariable("STELLAOPS_CONNECTOR_SIGNER_METADATA_PATH");
+    }
+
+    public bool Validate(AirgapImportRequest request, out string? errorCode, out string? message)
+    {
+        errorCode = null;
+        message = null;
+
+        if (string.IsNullOrWhiteSpace(_metadataPath) || !File.Exists(_metadataPath))
+        {
+            _logger.LogDebug("Airgap signer metadata not configured; skipping trust enforcement.");
+            return true;
+        }
+
+        _metadata ??= ConnectorSignerMetadataLoader.TryLoad(_metadataPath);
+        if (_metadata is null)
+        {
+            _logger.LogWarning("Failed to load airgap signer metadata from {Path}; allowing import.", _metadataPath);
+            return true;
+        }
+
+        if (string.IsNullOrWhiteSpace(request.Publisher))
+        {
+            errorCode = "AIRGAP_SOURCE_UNTRUSTED";
+            message = "publisher is required for trust enforcement.";
+            return false;
+        }
+
+        if (!_metadata.TryGet(request.Publisher, out var connector))
+        {
+            errorCode = "AIRGAP_SOURCE_UNTRUSTED";
+            message = $"Publisher '{request.Publisher}' is not present in trusted signer metadata.";
+            return false;
+        }
+
+        if (connector.Revoked)
+        {
+            errorCode = "AIRGAP_SOURCE_UNTRUSTED";
+            message = $"Publisher '{request.Publisher}' is revoked.";
+            return false;
+        }
+
+        if (connector.Bundle?.Digest is { } digest && !string.IsNullOrWhiteSpace(digest))
+        {
+            if (!string.Equals(digest.Trim(), request.PayloadHash?.Trim(), StringComparison.OrdinalIgnoreCase))
+            {
+                errorCode = "AIRGAP_PAYLOAD_MISMATCH";
+                message = "Payload hash does not match trusted bundle digest.";
+                return false;
+            }
+        }
+
+        // Basic sanity: ensure at least one signer entry exists.
+        if (connector.Signers.IsDefaultOrEmpty || connector.Signers.Sum(s => s.Fingerprints.Length) == 0)
+        {
+            errorCode = "AIRGAP_SOURCE_UNTRUSTED";
+            message = $"Publisher '{request.Publisher}' has no trusted signers configured.";
+            return false;
+        }
+
+        return true;
+    }
+}
+
--- a/src/Excititor/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj
+++ b/src/Excititor/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj
@@ -19,6 +19,7 @@
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj" />
+    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj" />
    <ProjectReference Include="../__Libraries/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj" />
--- a/src/Excititor/StellaOps.Excititor.WebService/Telemetry/ConsoleTelemetry.cs
+++ b/src/Excititor/StellaOps.Excititor.WebService/Telemetry/ConsoleTelemetry.cs
@@ -0,0 +1,15 @@
+using System.Diagnostics.Metrics;
+
+namespace StellaOps.Excititor.WebService.Telemetry;
+
+internal sealed class ConsoleTelemetry
+{
+    public const string MeterName = "StellaOps.Excititor.Console";
+
+    private static readonly Meter Meter = new(MeterName);
+
+    public Counter<long> Requests { get; } = Meter.CreateCounter<long>("console.vex.requests");
+    public Counter<long> CacheHits { get; } = Meter.CreateCounter<long>("console.vex.cache_hits");
+    public Counter<long> CacheMisses { get; } = Meter.CreateCounter<long>("console.vex.cache_misses");
+}
+