wip: doctor/cli/docs/api to vector db consolidation; api hardening for descriptions, tenant, and scopes; migrations and conversions of all DALs to EF v10

This commit is contained in:
master
2026-02-23 15:30:50 +02:00
parent bd8fee6ed8
commit e746577380
1424 changed files with 81225 additions and 25251 deletions

View File

@@ -1,323 +1,460 @@
// -----------------------------------------------------------------------------
// UnknownsEndpoints.cs
// Sprint: SPRINT_3600_0002_0001_unknowns_ranking_containment
// Task: UNK-RANK-007, UNK-RANK-008 - Implement GET /unknowns API with sorting/pagination
// Description: REST API for querying and filtering unknowns
// -----------------------------------------------------------------------------
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Routing;
using StellaOps.Unknowns.Core.Models;
using StellaOps.Unknowns.Core.Repositories;
using StellaOps.Unknowns.Core.Services;
using StellaOps.Scanner.WebService.Security;
using StellaOps.Scanner.WebService.Services;
using StellaOps.Scanner.WebService.Tenancy;
namespace StellaOps.Scanner.WebService.Endpoints;
internal static class UnknownsEndpoints
{
private const double HotBandThreshold = 0.70;
private const double WarmBandThreshold = 0.40;
private const string ExternalUnknownIdPrefix = "unk-";
public static void MapUnknownsEndpoints(this RouteGroupBuilder apiGroup)
{
var unknowns = apiGroup.MapGroup("/unknowns");
ArgumentNullException.ThrowIfNull(apiGroup);
var unknowns = apiGroup.MapGroup("/unknowns")
.WithTags("Unknowns")
.RequireAuthorization(ScannerPolicies.ScansRead);
unknowns.MapGet("/", HandleListAsync)
.WithName("scanner.unknowns.list")
.Produces<UnknownsListResponse>(StatusCodes.Status200OK)
.Produces<ProblemDetails>(StatusCodes.Status400BadRequest)
.WithDescription("List unknowns with optional sorting and filtering");
.Produces(StatusCodes.Status400BadRequest)
.WithDescription("Lists unknown entries with tenant-scoped filtering.");
unknowns.MapGet("/stats", HandleGetStatsAsync)
.WithName("scanner.unknowns.stats")
.Produces<UnknownsStatsResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status400BadRequest)
.WithDescription("Returns tenant-scoped unknown summary statistics.");
unknowns.MapGet("/bands", HandleGetBandsAsync)
.WithName("scanner.unknowns.bands")
.Produces<UnknownsBandsResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status400BadRequest)
.WithDescription("Returns tenant-scoped unknown distribution by triage band.");
unknowns.MapGet("/{id}/evidence", HandleGetEvidenceAsync)
.WithName("scanner.unknowns.evidence")
.Produces<UnknownEvidenceResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status400BadRequest)
.WithDescription("Returns tenant-scoped unknown evidence metadata.");
unknowns.MapGet("/{id}/history", HandleGetHistoryAsync)
.WithName("scanner.unknowns.history")
.Produces<UnknownHistoryResponse>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status400BadRequest)
.WithDescription("Returns tenant-scoped unknown history.");
unknowns.MapGet("/{id}", HandleGetByIdAsync)
.WithName("scanner.unknowns.get")
.Produces<UnknownDetailResponse>(StatusCodes.Status200OK)
.Produces<ProblemDetails>(StatusCodes.Status404NotFound)
.WithDescription("Get details of a specific unknown");
unknowns.MapGet("/{id}/proof", HandleGetProofAsync)
.WithName("scanner.unknowns.proof")
.Produces<UnknownProofResponse>(StatusCodes.Status200OK)
.Produces<ProblemDetails>(StatusCodes.Status404NotFound)
.WithDescription("Get the proof trail for an unknown ranking");
.Produces(StatusCodes.Status404NotFound)
.Produces(StatusCodes.Status400BadRequest)
.WithDescription("Returns tenant-scoped unknown detail.");
}
/// <summary>
/// GET /unknowns?sort=score&amp;order=desc&amp;artifact=sha256:...&amp;reason=missing_vex&amp;page=1&amp;limit=50
/// </summary>
private static async Task<IResult> HandleListAsync(
[FromQuery] string? sort,
[FromQuery] string? order,
[FromQuery] string? artifact,
[FromQuery] string? reason,
[FromQuery] string? kind,
[FromQuery] string? severity,
[FromQuery] double? minScore,
[FromQuery] double? maxScore,
[FromQuery] int? page,
[FromQuery] string? artifactDigest,
[FromQuery] string? vulnId,
[FromQuery] string? band,
[FromQuery] string? sortBy,
[FromQuery] string? sortOrder,
[FromQuery] int? limit,
IUnknownRepository repository,
IUnknownRanker ranker,
TimeProvider timeProvider,
[FromQuery] int? offset,
IUnknownsQueryService queryService,
HttpContext context,
CancellationToken cancellationToken)
{
// Validate and default pagination
var pageNum = Math.Max(1, page ?? 1);
var pageSize = Math.Clamp(limit ?? 50, 1, 200);
// Parse sort field
var sortField = (sort?.ToLowerInvariant()) switch
if (!TryResolveTenant(context, out var tenantId, out var failure))
{
"score" => UnknownSortField.Score,
"created" => UnknownSortField.Created,
"updated" => UnknownSortField.Updated,
"severity" => UnknownSortField.Severity,
"popularity" => UnknownSortField.Popularity,
_ => UnknownSortField.Score // Default to score
return failure!;
}
if (!TryMapBand(band, out var mappedBand))
{
return Results.BadRequest(new
{
type = "validation-error",
title = "Invalid band",
detail = "Band must be one of HOT, WARM, or COLD."
});
}
var query = new UnknownsListQuery
{
ArtifactDigest = string.IsNullOrWhiteSpace(artifactDigest) ? null : artifactDigest.Trim(),
VulnerabilityId = string.IsNullOrWhiteSpace(vulnId) ? null : vulnId.Trim(),
Band = mappedBand,
SortBy = MapSortField(sortBy),
SortOrder = MapSortOrder(sortOrder),
Limit = Math.Clamp(limit ?? 50, 1, 500),
Offset = Math.Max(offset ?? 0, 0)
};
var sortOrder = (order?.ToLowerInvariant()) switch
var result = await queryService.ListAsync(tenantId, query, cancellationToken).ConfigureAwait(false);
return Results.Ok(new UnknownsListResponse
{
"asc" => SortOrder.Ascending,
_ => SortOrder.Descending // Default to descending (highest first)
};
// Parse filters
UnknownKind? kindFilter = kind != null && Enum.TryParse<UnknownKind>(kind, true, out var k) ? k : null;
UnknownSeverity? severityFilter = severity != null && Enum.TryParse<UnknownSeverity>(severity, true, out var s) ? s : null;
var query = new UnknownListQuery(
ArtifactDigest: artifact,
Reason: reason,
Kind: kindFilter,
Severity: severityFilter,
MinScore: minScore,
MaxScore: maxScore,
SortField: sortField,
SortOrder: sortOrder,
Page: pageNum,
PageSize: pageSize);
var result = await repository.ListUnknownsAsync(query, cancellationToken);
var now = timeProvider.GetUtcNow();
return Results.Ok(new UnknownsListResponse(
Items: result.Items.Select(item => UnknownItemResponse.FromUnknownItem(item, now)).ToList(),
TotalCount: result.TotalCount,
Page: pageNum,
PageSize: pageSize,
TotalPages: (int)Math.Ceiling((double)result.TotalCount / pageSize),
HasNextPage: pageNum * pageSize < result.TotalCount,
HasPreviousPage: pageNum > 1));
Items = result.Items
.Select(MapItem)
.ToArray(),
TotalCount = result.TotalCount,
Limit = query.Limit,
Offset = query.Offset
});
}
/// <summary>
/// GET /unknowns/{id}
/// </summary>
private static async Task<IResult> HandleGetByIdAsync(
Guid id,
IUnknownRepository repository,
string id,
IUnknownsQueryService queryService,
HttpContext context,
CancellationToken cancellationToken)
{
var unknown = await repository.GetByIdAsync(id, cancellationToken);
if (unknown is null)
if (!TryResolveTenant(context, out var tenantId, out var failure))
{
return Results.NotFound(new ProblemDetails
{
Title = "Unknown not found",
Detail = $"No unknown found with ID: {id}",
Status = StatusCodes.Status404NotFound
});
return failure!;
}
return Results.Ok(UnknownDetailResponse.FromUnknown(unknown));
if (!TryParseUnknownId(id, out var unknownId))
{
return Results.NotFound();
}
var detail = await queryService.GetByIdAsync(tenantId, unknownId, cancellationToken).ConfigureAwait(false);
if (detail is null)
{
return Results.NotFound();
}
return Results.Ok(MapDetail(detail));
}
/// <summary>
/// GET /unknowns/{id}/proof
/// </summary>
private static async Task<IResult> HandleGetProofAsync(
Guid id,
IUnknownRepository repository,
private static async Task<IResult> HandleGetEvidenceAsync(
string id,
IUnknownsQueryService queryService,
HttpContext context,
CancellationToken cancellationToken)
{
var unknown = await repository.GetByIdAsync(id, cancellationToken);
if (unknown is null)
if (!TryResolveTenant(context, out var tenantId, out var failure))
{
return Results.NotFound(new ProblemDetails
{
Title = "Unknown not found",
Detail = $"No unknown found with ID: {id}",
Status = StatusCodes.Status404NotFound
});
return failure!;
}
var proofRef = unknown.ProofRef;
if (string.IsNullOrEmpty(proofRef))
if (!TryParseUnknownId(id, out var unknownId))
{
return Results.NotFound(new ProblemDetails
{
Title = "Proof not available",
Detail = $"No proof trail available for unknown: {id}",
Status = StatusCodes.Status404NotFound
});
return Results.NotFound();
}
// In a real implementation, read proof from storage
return Results.Ok(new UnknownProofResponse(
UnknownId: id,
ProofRef: proofRef,
CreatedAt: unknown.SysFrom));
var detail = await queryService.GetByIdAsync(tenantId, unknownId, cancellationToken).ConfigureAwait(false);
if (detail is null)
{
return Results.NotFound();
}
return Results.Ok(new UnknownEvidenceResponse
{
Id = ToExternalUnknownId(detail.UnknownId),
ProofRef = detail.ProofRef,
LastUpdatedAtUtc = detail.UpdatedAtUtc
});
}
private static async Task<IResult> HandleGetHistoryAsync(
string id,
IUnknownsQueryService queryService,
HttpContext context,
CancellationToken cancellationToken)
{
if (!TryResolveTenant(context, out var tenantId, out var failure))
{
return failure!;
}
if (!TryParseUnknownId(id, out var unknownId))
{
return Results.NotFound();
}
var detail = await queryService.GetByIdAsync(tenantId, unknownId, cancellationToken).ConfigureAwait(false);
if (detail is null)
{
return Results.NotFound();
}
return Results.Ok(new UnknownHistoryResponse
{
Id = ToExternalUnknownId(detail.UnknownId),
History = new[]
{
new UnknownHistoryEntryResponse
{
CapturedAtUtc = detail.UpdatedAtUtc,
Score = detail.Score,
Band = DetermineBand(detail.Score)
}
}
});
}
private static async Task<IResult> HandleGetStatsAsync(
IUnknownsQueryService queryService,
HttpContext context,
CancellationToken cancellationToken)
{
if (!TryResolveTenant(context, out var tenantId, out var failure))
{
return failure!;
}
var stats = await queryService.GetStatsAsync(tenantId, cancellationToken).ConfigureAwait(false);
return Results.Ok(new UnknownsStatsResponse
{
Total = stats.Total,
Hot = stats.Hot,
Warm = stats.Warm,
Cold = stats.Cold
});
}
private static async Task<IResult> HandleGetBandsAsync(
IUnknownsQueryService queryService,
HttpContext context,
CancellationToken cancellationToken)
{
if (!TryResolveTenant(context, out var tenantId, out var failure))
{
return failure!;
}
var distribution = await queryService.GetBandDistributionAsync(tenantId, cancellationToken).ConfigureAwait(false);
return Results.Ok(new UnknownsBandsResponse
{
Bands = distribution
});
}
private static bool TryResolveTenant(HttpContext context, out string tenantId, out IResult? failure)
{
tenantId = string.Empty;
failure = null;
if (ScannerRequestContextResolver.TryResolveTenant(
context,
out tenantId,
out var tenantError,
allowDefaultTenant: true))
{
return true;
}
failure = Results.BadRequest(new
{
type = "validation-error",
title = "Invalid tenant context",
detail = tenantError ?? "tenant_conflict"
});
return false;
}
private static UnknownsListItemResponse MapItem(UnknownsListItem item)
{
return new UnknownsListItemResponse
{
Id = ToExternalUnknownId(item.UnknownId),
ArtifactDigest = item.ArtifactDigest,
VulnerabilityId = item.VulnerabilityId,
PackagePurl = item.PackagePurl,
Score = item.Score,
Band = DetermineBand(item.Score),
CreatedAtUtc = item.CreatedAtUtc,
UpdatedAtUtc = item.UpdatedAtUtc
};
}
private static UnknownDetailResponse MapDetail(UnknownsDetail detail)
{
return new UnknownDetailResponse
{
Id = ToExternalUnknownId(detail.UnknownId),
ArtifactDigest = detail.ArtifactDigest,
VulnerabilityId = detail.VulnerabilityId,
PackagePurl = detail.PackagePurl,
Score = detail.Score,
Band = DetermineBand(detail.Score),
ProofRef = detail.ProofRef,
CreatedAtUtc = detail.CreatedAtUtc,
UpdatedAtUtc = detail.UpdatedAtUtc
};
}
private static UnknownsSortField MapSortField(string? rawValue)
{
if (string.IsNullOrWhiteSpace(rawValue))
{
return UnknownsSortField.Score;
}
return rawValue.Trim().ToLowerInvariant() switch
{
"score" => UnknownsSortField.Score,
"created" => UnknownsSortField.CreatedAt,
"createdat" => UnknownsSortField.CreatedAt,
"updated" => UnknownsSortField.UpdatedAt,
"updatedat" => UnknownsSortField.UpdatedAt,
"lastseen" => UnknownsSortField.UpdatedAt,
_ => UnknownsSortField.Score
};
}
private static UnknownsSortOrder MapSortOrder(string? rawValue)
{
if (string.IsNullOrWhiteSpace(rawValue))
{
return UnknownsSortOrder.Descending;
}
return rawValue.Trim().Equals("asc", StringComparison.OrdinalIgnoreCase)
? UnknownsSortOrder.Ascending
: UnknownsSortOrder.Descending;
}
private static bool TryMapBand(string? rawValue, out UnknownsBand? band)
{
band = null;
if (string.IsNullOrWhiteSpace(rawValue))
{
return true;
}
switch (rawValue.Trim().ToUpperInvariant())
{
case "HOT":
band = UnknownsBand.Hot;
return true;
case "WARM":
band = UnknownsBand.Warm;
return true;
case "COLD":
band = UnknownsBand.Cold;
return true;
default:
return false;
}
}
private static string DetermineBand(double score)
{
if (score >= HotBandThreshold)
{
return "HOT";
}
if (score >= WarmBandThreshold)
{
return "WARM";
}
return "COLD";
}
private static string ToExternalUnknownId(Guid unknownId)
=> $"{ExternalUnknownIdPrefix}{unknownId:N}";
private static bool TryParseUnknownId(string rawValue, out Guid unknownId)
{
unknownId = Guid.Empty;
if (string.IsNullOrWhiteSpace(rawValue))
{
return false;
}
var trimmed = rawValue.Trim();
if (Guid.TryParse(trimmed, out unknownId))
{
return true;
}
if (!trimmed.StartsWith(ExternalUnknownIdPrefix, StringComparison.OrdinalIgnoreCase))
{
return false;
}
var guidPart = trimmed[ExternalUnknownIdPrefix.Length..];
return Guid.TryParseExact(guidPart, "N", out unknownId)
|| Guid.TryParse(guidPart, out unknownId);
}
}
/// <summary>
/// Response model for unknowns list.
/// </summary>
public sealed record UnknownsListResponse(
IReadOnlyList<UnknownItemResponse> Items,
int TotalCount,
int Page,
int PageSize,
int TotalPages,
bool HasNextPage,
bool HasPreviousPage);
/// <summary>
/// Compact unknown item for list response.
/// </summary>
public sealed record UnknownItemResponse(
Guid Id,
string SubjectRef,
string Kind,
string? Severity,
double Score,
string TriageBand,
string Priority,
BlastRadiusResponse? BlastRadius,
ContainmentResponse? Containment,
DateTimeOffset CreatedAt)
public sealed record UnknownsListResponse
{
public static UnknownItemResponse FromUnknownItem(UnknownItem item, DateTimeOffset now) => new(
Id: Guid.TryParse(item.Id, out var id) ? id : Guid.Empty,
SubjectRef: item.ArtifactPurl ?? item.ArtifactDigest,
Kind: string.Join(",", item.Reasons),
Severity: null, // Would come from full Unknown
Score: item.Score,
TriageBand: item.Score.ToTriageBand().ToString(),
Priority: item.Score.ToPriorityLabel(),
BlastRadius: item.BlastRadius != null
? new BlastRadiusResponse(item.BlastRadius.Dependents, item.BlastRadius.NetFacing, item.BlastRadius.Privilege)
: null,
Containment: item.Containment != null
? new ContainmentResponse(item.Containment.Seccomp, item.Containment.Fs)
: null,
CreatedAt: now); // Would come from Unknown.SysFrom
public required IReadOnlyList<UnknownsListItemResponse> Items { get; init; }
public required int TotalCount { get; init; }
public required int Limit { get; init; }
public required int Offset { get; init; }
}
/// <summary>
/// Blast radius in API response.
/// </summary>
public sealed record BlastRadiusResponse(int Dependents, bool NetFacing, string Privilege);
/// <summary>
/// Containment signals in API response.
/// </summary>
public sealed record ContainmentResponse(string Seccomp, string Fs);
/// <summary>
/// Detailed unknown response.
/// </summary>
public sealed record UnknownDetailResponse(
Guid Id,
string TenantId,
string SubjectHash,
string SubjectType,
string SubjectRef,
string Kind,
string? Severity,
double Score,
string TriageBand,
double PopularityScore,
int DeploymentCount,
double UncertaintyScore,
BlastRadiusResponse? BlastRadius,
ContainmentResponse? Containment,
string? ProofRef,
DateTimeOffset ValidFrom,
DateTimeOffset? ValidTo,
DateTimeOffset SysFrom,
DateTimeOffset? ResolvedAt,
string? ResolutionType,
string? ResolutionRef)
public sealed record UnknownsListItemResponse
{
public static UnknownDetailResponse FromUnknown(Unknown u) => new(
Id: u.Id,
TenantId: u.TenantId,
SubjectHash: u.SubjectHash,
SubjectType: u.SubjectType.ToString(),
SubjectRef: u.SubjectRef,
Kind: u.Kind.ToString(),
Severity: u.Severity?.ToString(),
Score: u.TriageScore,
TriageBand: u.TriageScore.ToTriageBand().ToString(),
PopularityScore: u.PopularityScore,
DeploymentCount: u.DeploymentCount,
UncertaintyScore: u.UncertaintyScore,
BlastRadius: u.BlastDependents.HasValue
? new BlastRadiusResponse(u.BlastDependents.Value, u.BlastNetFacing ?? false, u.BlastPrivilege ?? "user")
: null,
Containment: !string.IsNullOrEmpty(u.ContainmentSeccomp) || !string.IsNullOrEmpty(u.ContainmentFs)
? new ContainmentResponse(u.ContainmentSeccomp ?? "unknown", u.ContainmentFs ?? "unknown")
: null,
ProofRef: u.ProofRef,
ValidFrom: u.ValidFrom,
ValidTo: u.ValidTo,
SysFrom: u.SysFrom,
ResolvedAt: u.ResolvedAt,
ResolutionType: u.ResolutionType?.ToString(),
ResolutionRef: u.ResolutionRef);
public required string Id { get; init; }
public required string ArtifactDigest { get; init; }
public required string VulnerabilityId { get; init; }
public required string PackagePurl { get; init; }
public required double Score { get; init; }
public required string Band { get; init; }
public required DateTimeOffset CreatedAtUtc { get; init; }
public required DateTimeOffset UpdatedAtUtc { get; init; }
}
/// <summary>
/// Proof trail response.
/// </summary>
public sealed record UnknownProofResponse(
Guid UnknownId,
string ProofRef,
DateTimeOffset CreatedAt);
/// <summary>
/// Sort fields for unknowns query.
/// </summary>
public enum UnknownSortField
public sealed record UnknownDetailResponse
{
Score,
Created,
Updated,
Severity,
Popularity
public required string Id { get; init; }
public required string ArtifactDigest { get; init; }
public required string VulnerabilityId { get; init; }
public required string PackagePurl { get; init; }
public required double Score { get; init; }
public required string Band { get; init; }
public string? ProofRef { get; init; }
public required DateTimeOffset CreatedAtUtc { get; init; }
public required DateTimeOffset UpdatedAtUtc { get; init; }
}
/// <summary>
/// Sort order.
/// </summary>
public enum SortOrder
public sealed record UnknownEvidenceResponse
{
Ascending,
Descending
public required string Id { get; init; }
public string? ProofRef { get; init; }
public required DateTimeOffset LastUpdatedAtUtc { get; init; }
}
/// <summary>
/// Query parameters for listing unknowns.
/// </summary>
public sealed record UnknownListQuery(
string? ArtifactDigest,
string? Reason,
UnknownKind? Kind,
UnknownSeverity? Severity,
double? MinScore,
double? MaxScore,
UnknownSortField SortField,
SortOrder SortOrder,
int Page,
int PageSize);
public sealed record UnknownHistoryResponse
{
public required string Id { get; init; }
public required IReadOnlyList<UnknownHistoryEntryResponse> History { get; init; }
}
public sealed record UnknownHistoryEntryResponse
{
public required DateTimeOffset CapturedAtUtc { get; init; }
public required double Score { get; init; }
public required string Band { get; init; }
}
public sealed record UnknownsStatsResponse
{
public required long Total { get; init; }
public required long Hot { get; init; }
public required long Warm { get; init; }
public required long Cold { get; init; }
}
public sealed record UnknownsBandsResponse
{
public required IReadOnlyDictionary<string, long> Bands { get; init; }
}