stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Details
Findings Ledger CI / build-test (push) Has been cancelled
Details
Findings Ledger CI / migration-validation (push) Has been cancelled
Details
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Details
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Details
AOC Guard CI / aoc-guard (push) Has been cancelled
Details
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Details
cryptopro-linux-csp / build-and-test (push) Has been cancelled
Details
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Details
Signals CI & Image / signals-ci (push) Has been cancelled
Details
sm-remote-ci / build-and-test (push) Has been cancelled
Details
Findings Ledger CI / generate-manifest (push) Has been cancelled
Details
AOC Guard CI / aoc-verify (push) Has been cancelled
Details
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Details
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Details
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
Details

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-09 09:38:09 +02:00
parent bc0762e97d
commit 108d1c64b3
193 changed files with 7265 additions and 13029 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/Excititor/StellaOps.Excititor.WebService/AGENTS.md
View File

@@ -27,14 +27,15 @@ Expose Excititor APIs (console VEX views, graph/Vuln Explorer feeds, observation
5. Observability: structured logs, counters, optional OTEL traces behind configuration flags.
## Testing
- Prefer deterministic API/integration tests under `__Tests` with seeded Mongo fixtures.
- Prefer deterministic API/integration tests under `__Tests` with seeded Postgres fixtures or in-memory stores.
- Verify RBAC/tenant isolation, idempotent ingestion, and stable ordering of VEX aggregates.
- Use ISO-8601 UTC timestamps and stable sorting in responses; assert on content hashes where applicable.
## Determinism & Data
- MongoDB is the canonical store; never apply consensus transformations before persistence.
- Postgres append-only storage is canonical; never apply consensus transformations before persistence.
- Ensure paged/list endpoints use explicit sort keys (e.g., vendor, upstreamId, version, createdUtc).
- Avoid nondeterministic clocks/randomness; inject clocks and GUID providers for tests.
- Evidence/attestation endpoints are temporarily disabled; re-enable only when Postgres-backed stores land (Mongo/BSON removed).
## Boundaries
- Do not modify Policy Engine or Cartographer schemas from here; consume published contracts only.

319
src/Excititor/StellaOps.Excititor.WebService/Endpoints/AttestationEndpoints.cs
View File

@@ -1,40 +1,23 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
namespace StellaOps.Excititor.WebService.Endpoints;
/// <summary>
/// Attestation API endpoints (WEB-OBS-54-001).
/// Exposes /attestations/vex/* endpoints returning DSSE verification state,
/// builder identity, and chain-of-custody links.
/// Attestation API endpoints (temporarily disabled while Mongo is removed and Postgres storage is adopted).
/// </summary>
public static class AttestationEndpoints
{
public static void MapAttestationEndpoints(this WebApplication app)
{
// GET /attestations/vex/list - List attestations
app.MapGet("/attestations/vex/list", async (
// GET /attestations/vex/list
app.MapGet("/attestations/vex/list", (
HttpContext context,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
[FromQuery] string? cursor,
[FromQuery] string? vulnerabilityId,
[FromQuery] string? productKey,
CancellationToken cancellationToken) =>
IOptions<VexStorageOptions> storageOptions) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -42,70 +25,22 @@ public static class AttestationEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200);
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Attestations);
var builder = Builders<BsonDocument>.Filter;
var filters = new List<FilterDefinition<BsonDocument>>();
if (!string.IsNullOrWhiteSpace(vulnerabilityId))
{
filters.Add(builder.Eq("VulnerabilityId", vulnerabilityId.Trim().ToUpperInvariant()));
}
if (!string.IsNullOrWhiteSpace(productKey))
{
filters.Add(builder.Eq("ProductKey", productKey.Trim().ToLowerInvariant()));
}
// Parse cursor if provided
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
{
var ltTime = builder.Lt("IssuedAt", cursorTime);
var eqTimeLtId = builder.And(
builder.Eq("IssuedAt", cursorTime),
builder.Lt("_id", cursorId));
filters.Add(builder.Or(ltTime, eqTimeLtId));
}
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
var sort = Builders<BsonDocument>.Sort.Descending("IssuedAt").Descending("_id");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var items = documents.Select(doc => ToListItem(doc, tenant, timeProvider)).ToList();
string? nextCursor = null;
var hasMore = documents.Count == take;
if (hasMore && documents.Count > 0)
{
var last = documents[^1];
var lastTime = last.GetValue("IssuedAt", BsonNull.Value).ToUniversalTime();
var lastId = last.GetValue("_id", BsonNull.Value).AsString;
nextCursor = EncodeCursor(lastTime, lastId);
}
var response = new VexAttestationListResponse(items, nextCursor, hasMore, items.Count);
return Results.Ok(response);
return Results.Problem(
detail: "Attestation listing is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("ListVexAttestations");
// GET /attestations/vex/{attestationId} - Get attestation details
app.MapGet("/attestations/vex/{attestationId}", async (
// GET /attestations/vex/{attestationId}
app.MapGet("/attestations/vex/{attestationId}", (
HttpContext context,
string attestationId,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexAttestationLinkStore attestationStore,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
IOptions<VexStorageOptions> storageOptions) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -113,235 +48,23 @@ public static class AttestationEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(attestationId))
{
return Results.BadRequest(new { error = new { code = "ERR_ATTESTATION_ID", message = "attestationId is required" } });
return Results.Problem(
detail: "attestationId is required.",
statusCode: StatusCodes.Status400BadRequest,
title: "Validation error");
}
var attestation = await attestationStore.FindAsync(attestationId.Trim(), cancellationToken).ConfigureAwait(false);
if (attestation is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = $"Attestation '{attestationId}' not found" } });
}
// Build subject from observation context
var subjectDigest = attestation.Metadata.TryGetValue("digest", out var dig) ? dig : attestation.ObservationId;
var subject = new VexAttestationSubject(
Digest: subjectDigest,
DigestAlgorithm: "sha256",
Name: $"{attestation.VulnerabilityId}/{attestation.ProductKey}",
Uri: null);
var builder = new VexAttestationBuilderIdentity(
Id: attestation.SupplierId,
Version: null,
BuilderId: attestation.SupplierId,
InvocationId: attestation.ObservationId);
// Get verification state from metadata
var isValid = attestation.Metadata.TryGetValue("verified", out var verified) && verified == "true";
DateTimeOffset? verifiedAt = null;
if (attestation.Metadata.TryGetValue("verifiedAt", out var verifiedAtStr) &&
DateTimeOffset.TryParse(verifiedAtStr, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsedVerifiedAt))
{
verifiedAt = parsedVerifiedAt;
}
var verification = new VexAttestationVerificationState(
Valid: isValid,
VerifiedAt: verifiedAt,
SignatureType: attestation.Metadata.GetValueOrDefault("signatureType", "dsse"),
KeyId: attestation.Metadata.GetValueOrDefault("keyId"),
Issuer: attestation.Metadata.GetValueOrDefault("issuer"),
EnvelopeDigest: attestation.Metadata.GetValueOrDefault("envelopeDigest"),
Diagnostics: attestation.Metadata);
var custodyLinks = new List<VexAttestationCustodyLink>
{
new(
Step: 1,
Actor: attestation.SupplierId,
Action: "created",
Timestamp: attestation.IssuedAt,
Reference: attestation.AttestationId)
};
// Add linkset link
custodyLinks.Add(new VexAttestationCustodyLink(
Step: 2,
Actor: "excititor",
Action: "linked_to_observation",
Timestamp: attestation.IssuedAt,
Reference: attestation.LinksetId));
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
{
["observationId"] = attestation.ObservationId,
["linksetId"] = attestation.LinksetId,
["vulnerabilityId"] = attestation.VulnerabilityId,
["productKey"] = attestation.ProductKey
};
if (!string.IsNullOrWhiteSpace(attestation.JustificationSummary))
{
metadata["justificationSummary"] = attestation.JustificationSummary;
}
var response = new VexAttestationDetailResponse(
AttestationId: attestation.AttestationId,
Tenant: tenant,
CreatedAt: attestation.IssuedAt,
PredicateType: attestation.Metadata.GetValueOrDefault("predicateType", "https://in-toto.io/attestation/v1"),
Subject: subject,
Builder: builder,
Verification: verification,
ChainOfCustody: custodyLinks,
Metadata: metadata);
return Results.Ok(response);
return Results.Problem(
detail: "Attestation retrieval is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("GetVexAttestation");
// GET /attestations/vex/lookup - Lookup attestations by linkset or observation
app.MapGet("/attestations/vex/lookup", async (
HttpContext context,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] string? linksetId,
[FromQuery] string? observationId,
[FromQuery] int? limit,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(linksetId) && string.IsNullOrWhiteSpace(observationId))
{
return Results.BadRequest(new { error = new { code = "ERR_PARAMS", message = "Either linksetId or observationId is required" } });
}
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 100);
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Attestations);
var builder = Builders<BsonDocument>.Filter;
FilterDefinition<BsonDocument> filter;
if (!string.IsNullOrWhiteSpace(linksetId))
{
filter = builder.Eq("LinksetId", linksetId.Trim());
}
else
{
filter = builder.Eq("ObservationId", observationId!.Trim());
}
var sort = Builders<BsonDocument>.Sort.Descending("IssuedAt");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var items = documents.Select(doc => ToListItem(doc, tenant, timeProvider)).ToList();
var response = new VexAttestationLookupResponse(
SubjectDigest: linksetId ?? observationId ?? string.Empty,
Attestations: items,
QueriedAt: timeProvider.GetUtcNow());
return Results.Ok(response);
}).WithName("LookupVexAttestations");
}
private static VexAttestationListItem ToListItem(BsonDocument doc, string tenant, TimeProvider timeProvider)
{
return new VexAttestationListItem(
AttestationId: doc.GetValue("_id", BsonNull.Value).AsString ?? string.Empty,
Tenant: tenant,
CreatedAt: doc.GetValue("IssuedAt", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["IssuedAt"].ToUniversalTime(), TimeSpan.Zero)
: timeProvider.GetUtcNow(),
PredicateType: "https://in-toto.io/attestation/v1",
SubjectDigest: doc.GetValue("ObservationId", BsonNull.Value).AsString ?? string.Empty,
Valid: doc.Contains("Metadata") && !doc["Metadata"].IsBsonNull &&
doc["Metadata"].AsBsonDocument.Contains("verified") &&
doc["Metadata"]["verified"].AsString == "true",
BuilderId: doc.GetValue("SupplierId", BsonNull.Value).AsString);
}
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, out string tenant, out IResult? problem)
{
tenant = options.DefaultTenant;
problem = null;
if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerValues) && headerValues.Count > 0)
{
var requestedTenant = headerValues[0]?.Trim();
if (string.IsNullOrEmpty(requestedTenant))
{
problem = Results.BadRequest(new { error = new { code = "ERR_TENANT", message = "X-Stella-Tenant header must not be empty" } });
return false;
}
if (!string.Equals(requestedTenant, options.DefaultTenant, StringComparison.OrdinalIgnoreCase))
{
problem = Results.Json(
new { error = new { code = "ERR_TENANT_FORBIDDEN", message = $"Tenant '{requestedTenant}' is not allowed" } },
statusCode: StatusCodes.Status403Forbidden);
return false;
}
tenant = requestedTenant;
}
return true;
}
private static bool TryDecodeCursor(string cursor, out DateTime timestamp, out string id)
{
timestamp = default;
id = string.Empty;
try
{
var payload = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(cursor));
var parts = payload.Split('|');
if (parts.Length != 2)
{
return false;
}
if (!DateTimeOffset.TryParse(parts[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
{
return false;
}
timestamp = parsed.UtcDateTime;
id = parts[1];
return true;
}
catch
{
return false;
}
}
private static string EncodeCursor(DateTime timestamp, string id)
{
var payload = FormattableString.Invariant($"{timestamp:O}|{id}");
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(payload));
}
}

744
src/Excititor/StellaOps.Excititor.WebService/Endpoints/EvidenceEndpoints.cs
View File

@@ -1,48 +1,24 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.IO;
using System.Threading.Tasks;
using System.Security.Cryptography;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Canonicalization;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Services;
using StellaOps.Excititor.WebService.Telemetry;
using StellaOps.Excititor.WebService.Options;
namespace StellaOps.Excititor.WebService.Endpoints;
/// <summary>
/// Evidence API endpoints (WEB-OBS-53-001).
/// Exposes /evidence/vex/* endpoints that fetch locker bundles, enforce scopes,
/// and surface verification metadata without synthesizing verdicts.
/// Evidence API endpoints (temporarily disabled while Mongo/BSON storage is removed).
/// </summary>
public static class EvidenceEndpoints
{
public static void MapEvidenceEndpoints(this WebApplication app)
{
// GET /evidence/vex/list - List evidence exports
app.MapGet("/evidence/vex/list", async (
// GET /evidence/vex/list
app.MapGet("/evidence/vex/list", (
HttpContext context,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
[FromQuery] string? cursor,
[FromQuery] string? format,
CancellationToken cancellationToken) =>
ChunkTelemetry chunkTelemetry) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -50,74 +26,23 @@ public static class EvidenceEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200);
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Exports);
var builder = Builders<BsonDocument>.Filter;
var filters = new List<FilterDefinition<BsonDocument>>();
if (!string.IsNullOrWhiteSpace(format))
{
filters.Add(builder.Eq("Format", format.Trim().ToLowerInvariant()));
}
// Parse cursor if provided (base64-encoded timestamp|id)
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
{
var ltTime = builder.Lt("CreatedAt", cursorTime);
var eqTimeLtId = builder.And(
builder.Eq("CreatedAt", cursorTime),
builder.Lt("_id", cursorId));
filters.Add(builder.Or(ltTime, eqTimeLtId));
}
var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
var sort = Builders<BsonDocument>.Sort.Descending("CreatedAt").Descending("_id");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var items = documents.Select(doc => new VexEvidenceListItem(
BundleId: doc.GetValue("ExportId", BsonNull.Value).AsString ?? doc.GetValue("_id", BsonNull.Value).AsString,
Tenant: tenant,
CreatedAt: doc.GetValue("CreatedAt", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["CreatedAt"].ToUniversalTime(), TimeSpan.Zero)
: timeProvider.GetUtcNow(),
ContentHash: doc.GetValue("ArtifactDigest", BsonNull.Value).AsString ?? string.Empty,
Format: doc.GetValue("Format", BsonNull.Value).AsString ?? "json",
ItemCount: doc.GetValue("ClaimCount", BsonNull.Value).IsInt32 ? doc["ClaimCount"].AsInt32 : 0,
Verified: doc.Contains("Attestation") && !doc["Attestation"].IsBsonNull)).ToList();
string? nextCursor = null;
var hasMore = documents.Count == take;
if (hasMore && documents.Count > 0)
{
var last = documents[^1];
var lastTime = last.GetValue("CreatedAt", BsonNull.Value).ToUniversalTime();
var lastId = last.GetValue("_id", BsonNull.Value).AsString;
nextCursor = EncodeCursor(lastTime, lastId);
}
var response = new VexEvidenceListResponse(items, nextCursor, hasMore, items.Count);
return Results.Ok(response);
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
return Results.Problem(
detail: "Evidence exports are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("ListVexEvidence");
// GET /evidence/vex/bundle/{bundleId} - Get evidence bundle details
app.MapGet("/evidence/vex/bundle/{bundleId}", async (
// GET /evidence/vex/{bundleId}
app.MapGet("/evidence/vex/{bundleId}", (
HttpContext context,
string bundleId,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
IOptions<VexStorageOptions> storageOptions) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -125,79 +50,30 @@ public static class EvidenceEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out _, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(bundleId))
{
return Results.BadRequest(new { error = new { code = "ERR_BUNDLE_ID", message = "bundleId is required" } });
return Results.Problem(
detail: "bundleId is required.",
statusCode: StatusCodes.Status400BadRequest,
title: "Validation error");
}
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Exports);
var filter = Builders<BsonDocument>.Filter.Or(
Builders<BsonDocument>.Filter.Eq("_id", bundleId.Trim()),
Builders<BsonDocument>.Filter.Eq("ExportId", bundleId.Trim()));
var doc = await collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (doc is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = $"Evidence bundle '{bundleId}' not found" } });
}
VexEvidenceVerificationMetadata? verification = null;
if (doc.Contains("Attestation") && !doc["Attestation"].IsBsonNull)
{
var att = doc["Attestation"].AsBsonDocument;
verification = new VexEvidenceVerificationMetadata(
Verified: true,
VerifiedAt: att.Contains("SignedAt") && att["SignedAt"].IsBsonDateTime
? new DateTimeOffset(att["SignedAt"].ToUniversalTime(), TimeSpan.Zero)
: null,
SignatureType: "dsse",
KeyId: att.GetValue("KeyId", BsonNull.Value).AsString,
Issuer: att.GetValue("Issuer", BsonNull.Value).AsString,
TransparencyRef: att.Contains("Rekor") && !att["Rekor"].IsBsonNull
? att["Rekor"].AsBsonDocument.GetValue("Location", BsonNull.Value).AsString
: null);
}
var metadata = new Dictionary<string, string>(StringComparer.Ordinal);
if (doc.Contains("SourceProviders") && doc["SourceProviders"].IsBsonArray)
{
metadata["sourceProviders"] = string.Join(",", doc["SourceProviders"].AsBsonArray.Select(v => v.AsString));
}
if (doc.Contains("PolicyRevisionId") && !doc["PolicyRevisionId"].IsBsonNull)
{
metadata["policyRevisionId"] = doc["PolicyRevisionId"].AsString;
}
var response = new VexEvidenceBundleResponse(
BundleId: doc.GetValue("ExportId", BsonNull.Value).AsString ?? bundleId.Trim(),
Tenant: tenant,
CreatedAt: doc.GetValue("CreatedAt", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["CreatedAt"].ToUniversalTime(), TimeSpan.Zero)
: timeProvider.GetUtcNow(),
ContentHash: doc.GetValue("ArtifactDigest", BsonNull.Value).AsString ?? string.Empty,
Format: doc.GetValue("Format", BsonNull.Value).AsString ?? "json",
ItemCount: doc.GetValue("ClaimCount", BsonNull.Value).IsInt32 ? doc["ClaimCount"].AsInt32 : 0,
Verification: verification,
Metadata: metadata);
return Results.Ok(response);
return Results.Problem(
detail: "Evidence bundles are temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("GetVexEvidenceBundle");
// GET /evidence/vex/lookup - Lookup evidence for vuln/product pair
app.MapGet("/evidence/vex/lookup", async (
// GET /v1/vex/evidence/chunks
app.MapGet("/v1/vex/evidence/chunks", (
HttpContext context,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IVexObservationProjectionService projectionService,
TimeProvider timeProvider,
[FromQuery] string vulnerabilityId,
[FromQuery] string productKey,
[FromQuery] int? limit,
CancellationToken cancellationToken) =>
ChunkTelemetry chunkTelemetry) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
@@ -205,572 +81,16 @@ public static class EvidenceEndpoints
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
{
return Results.BadRequest(new { error = new { code = "ERR_PARAMS", message = "vulnerabilityId and productKey are required" } });
}
var take = Math.Clamp(limit.GetValueOrDefault(100), 1, 500);
var request = new VexObservationProjectionRequest(
tenant,
vulnerabilityId.Trim(),
productKey.Trim(),
ImmutableHashSet<string>.Empty,
ImmutableHashSet<VexClaimStatus>.Empty,
null,
take);
var result = await projectionService.QueryAsync(request, cancellationToken).ConfigureAwait(false);
var items = result.Statements.Select(s => new VexEvidenceItem(
ObservationId: s.ObservationId,
ProviderId: s.ProviderId,
Status: s.Status.ToString().ToLowerInvariant(),
Justification: s.Justification?.ToString().ToLowerInvariant(),
FirstSeen: s.FirstSeen,
LastSeen: s.LastSeen,
DocumentDigest: s.Document.Digest,
Verification: s.Signature is null ? null : new VexEvidenceVerificationMetadata(
Verified: s.Signature.VerifiedAt.HasValue,
VerifiedAt: s.Signature.VerifiedAt,
SignatureType: s.Signature.Type,
KeyId: s.Signature.KeyId,
Issuer: s.Signature.Issuer,
TransparencyRef: null))).ToList();
var response = new VexEvidenceLookupResponse(
VulnerabilityId: vulnerabilityId.Trim(),
ProductKey: productKey.Trim(),
EvidenceItems: items,
QueriedAt: timeProvider.GetUtcNow());
return Results.Ok(response);
}).WithName("LookupVexEvidence");
// GET /vuln/evidence/vex/{advisory_key} - Get evidence by advisory key (EXCITITOR-VULN-29-002)
app.MapGet("/vuln/evidence/vex/{advisory_key}", async (
HttpContext context,
string advisory_key,
IOptions<VexStorageOptions> storageOptions,
[FromServices] IMongoDatabase database,
TimeProvider timeProvider,
[FromQuery] int? limit,
[FromQuery] string? cursor,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(advisory_key))
{
NormalizationTelemetry.RecordAdvisoryKeyCanonicalizeError(tenant, "empty_key");
return Results.BadRequest(new { error = new { code = "ERR_ADVISORY_KEY", message = "advisory_key is required" } });
}
var stopwatch = Stopwatch.StartNew();
// Canonicalize the advisory key using VexAdvisoryKeyCanonicalizer
var canonicalizer = new VexAdvisoryKeyCanonicalizer();
VexCanonicalAdvisoryKey canonicalKey;
try
{
canonicalKey = canonicalizer.Canonicalize(advisory_key.Trim());
NormalizationTelemetry.RecordAdvisoryKeyCanonicalization(tenant, canonicalKey);
}
catch (ArgumentException ex)
{
NormalizationTelemetry.RecordAdvisoryKeyCanonicalizeError(tenant, "invalid_format", advisory_key);
return Results.BadRequest(new { error = new { code = "ERR_INVALID_ADVISORY_KEY", message = ex.Message } });
}
var take = Math.Clamp(limit.GetValueOrDefault(100), 1, 500);
var collection = database.GetCollection<BsonDocument>(VexMongoCollectionNames.Statements);
var builder = Builders<BsonDocument>.Filter;
// Build filter to match by vulnerability ID (case-insensitive)
// Try original key, canonical key, and all aliases
var vulnerabilityFilters = new List<FilterDefinition<BsonDocument>>
{
builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(advisory_key.Trim())}$", "i"))
};
// Add canonical key if different
if (!string.Equals(canonicalKey.AdvisoryKey, advisory_key.Trim(), StringComparison.OrdinalIgnoreCase))
{
vulnerabilityFilters.Add(builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(canonicalKey.AdvisoryKey)}$", "i")));
}
// Add original ID if available
if (canonicalKey.OriginalId is { } originalId &&
!string.Equals(originalId, advisory_key.Trim(), StringComparison.OrdinalIgnoreCase))
{
vulnerabilityFilters.Add(builder.Regex("VulnerabilityId", new BsonRegularExpression($"^{EscapeRegex(originalId)}$", "i")));
}
var filter = builder.Or(vulnerabilityFilters);
// Apply cursor-based pagination if provided
if (!string.IsNullOrWhiteSpace(cursor) && TryDecodeCursor(cursor, out var cursorTime, out var cursorId))
{
var ltTime = builder.Lt("InsertedAt", cursorTime);
var eqTimeLtId = builder.And(
builder.Eq("InsertedAt", cursorTime),
builder.Lt("_id", ObjectId.Parse(cursorId)));
filter = builder.And(filter, builder.Or(ltTime, eqTimeLtId));
}
var sort = Builders<BsonDocument>.Sort.Descending("InsertedAt").Descending("_id");
var documents = await collection
.Find(filter)
.Sort(sort)
.Limit(take)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
var now = timeProvider.GetUtcNow();
var statements = new List<VexAdvisoryStatementResponse>();
foreach (var doc in documents)
{
var provenance = new VexAdvisoryProvenanceResponse(
DocumentDigest: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
? doc["Document"].AsBsonDocument.GetValue("Digest", BsonNull.Value).AsString ?? string.Empty
: string.Empty,
DocumentFormat: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
? doc["Document"].AsBsonDocument.GetValue("Format", BsonNull.Value).AsString ?? "unknown"
: "unknown",
SourceUri: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
? doc["Document"].AsBsonDocument.GetValue("SourceUri", BsonNull.Value).AsString ?? string.Empty
: string.Empty,
Revision: doc.GetValue("Document", BsonNull.Value).IsBsonDocument
? doc["Document"].AsBsonDocument.GetValue("Revision", BsonNull.Value).AsString
: null,
InsertedAt: doc.GetValue("InsertedAt", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["InsertedAt"].ToUniversalTime(), TimeSpan.Zero)
: now);
VexAdvisoryAttestationResponse? attestation = null;
if (doc.GetValue("Document", BsonNull.Value).IsBsonDocument)
{
var docSection = doc["Document"].AsBsonDocument;
if (docSection.Contains("Signature") && !docSection["Signature"].IsBsonNull)
{
var sig = docSection["Signature"].AsBsonDocument;
var sigType = sig.GetValue("Type", BsonNull.Value).AsString;
if (!string.IsNullOrWhiteSpace(sigType))
{
attestation = new VexAdvisoryAttestationResponse(
SignatureType: sigType,
Issuer: sig.GetValue("Issuer", BsonNull.Value).AsString,
Subject: sig.GetValue("Subject", BsonNull.Value).AsString,
KeyId: sig.GetValue("KeyId", BsonNull.Value).AsString,
VerifiedAt: sig.Contains("VerifiedAt") && !sig["VerifiedAt"].IsBsonNull
? new DateTimeOffset(sig["VerifiedAt"].ToUniversalTime(), TimeSpan.Zero)
: null,
TransparencyLogRef: sig.GetValue("TransparencyLogReference", BsonNull.Value).AsString,
TrustWeight: sig.Contains("TrustWeight") && !sig["TrustWeight"].IsBsonNull
? (decimal)sig["TrustWeight"].ToDouble()
: null,
TrustTier: DeriveTrustTier(sig.GetValue("TrustIssuerId", BsonNull.Value).AsString));
}
}
}
var productDoc = doc.GetValue("Product", BsonNull.Value).IsBsonDocument
? doc["Product"].AsBsonDocument
: null;
var product = new VexAdvisoryProductResponse(
Key: productDoc?.GetValue("Key", BsonNull.Value).AsString ?? string.Empty,
Name: productDoc?.GetValue("Name", BsonNull.Value).AsString,
Version: productDoc?.GetValue("Version", BsonNull.Value).AsString,
Purl: productDoc?.GetValue("Purl", BsonNull.Value).AsString,
Cpe: productDoc?.GetValue("Cpe", BsonNull.Value).AsString);
statements.Add(new VexAdvisoryStatementResponse(
StatementId: doc.GetValue("_id", BsonNull.Value).ToString() ?? string.Empty,
ProviderId: doc.GetValue("ProviderId", BsonNull.Value).AsString ?? string.Empty,
Product: product,
Status: doc.GetValue("Status", BsonNull.Value).AsString ?? "unknown",
Justification: doc.GetValue("Justification", BsonNull.Value).AsString,
Detail: doc.GetValue("Detail", BsonNull.Value).AsString,
FirstSeen: doc.GetValue("FirstSeen", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["FirstSeen"].ToUniversalTime(), TimeSpan.Zero)
: now,
LastSeen: doc.GetValue("LastSeen", BsonNull.Value).IsBsonDateTime
? new DateTimeOffset(doc["LastSeen"].ToUniversalTime(), TimeSpan.Zero)
: now,
Provenance: provenance,
Attestation: attestation));
}
var aliases = canonicalKey.Links
.Select(link => new VexAdvisoryLinkResponse(link.Identifier, link.Type, link.IsOriginal))
.ToList();
stopwatch.Stop();
NormalizationTelemetry.RecordEvidenceRetrieval(
tenant,
"success",
statements.Count,
stopwatch.Elapsed.TotalSeconds);
var response = new VexAdvisoryEvidenceResponse(
AdvisoryKey: advisory_key.Trim(),
CanonicalKey: canonicalKey.AdvisoryKey,
Scope: canonicalKey.Scope.ToString().ToLowerInvariant(),
Aliases: aliases,
Statements: statements,
QueriedAt: now,
TotalCount: statements.Count);
return Results.Ok(response);
}).WithName("GetVexAdvisoryEvidence");
// GET /evidence/vex/locker/{bundleId}
app.MapGet("/evidence/vex/locker/{bundleId}", async (
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
[FromServices] IVexHashingService hashingService,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
if (string.IsNullOrWhiteSpace(bundleId))
{
return Results.BadRequest(new { error = new { code = "ERR_BUNDLE_ID", message = "bundleId is required" } });
}
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
.ConfigureAwait(false);
if (record is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Locker manifest not found" } });
}
// Optional local hash/size computation when locker root is configured
long? manifestSize = null;
long? evidenceSize = null;
string? evidenceHash = null;
var lockerRoot = airgapOptions.Value.LockerRootPath;
if (!string.IsNullOrWhiteSpace(lockerRoot))
{
TryHashFile(lockerRoot, record.PortableManifestPath, hashingService, out var manifestHash, out manifestSize);
if (!string.IsNullOrWhiteSpace(manifestHash))
{
record.PortableManifestHash = manifestHash!;
}
TryHashFile(lockerRoot, record.EvidenceLockerPath, hashingService, out evidenceHash, out evidenceSize);
}
var timeline = record.Timeline
.OrderBy(entry => entry.CreatedAt)
.Select(entry => new VexEvidenceLockerTimelineEntry(
entry.EventType,
entry.CreatedAt,
entry.ErrorCode,
entry.Message,
entry.StalenessSeconds))
.ToList();
var response = new VexEvidenceLockerResponse(
record.BundleId,
record.MirrorGeneration,
record.TenantId,
record.Publisher,
record.PayloadHash,
record.PortableManifestPath,
record.PortableManifestHash,
record.EvidenceLockerPath,
evidenceHash,
manifestSize,
evidenceSize,
record.ImportedAt,
record.Timeline.FirstOrDefault()?.StalenessSeconds,
record.TransparencyLog,
timeline);
return Results.Ok(response);
}).WithName("GetVexEvidenceLockerManifest");
// GET /evidence/vex/locker/{bundleId}/manifest/file
app.MapGet("/evidence/vex/locker/{bundleId}/manifest/file", async (
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
var root = airgapOptions.Value.LockerRootPath;
if (string.IsNullOrWhiteSpace(root))
{
return Results.NotFound(new { error = new { code = "ERR_LOCKER_ROOT", message = "LockerRootPath is not configured" } });
}
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
.ConfigureAwait(false);
if (record is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Locker manifest not found" } });
}
if (!TryResolveLockerFile(root, record.PortableManifestPath, out var fullPath))
{
return Results.NotFound(new { error = new { code = "ERR_MANIFEST_FILE", message = "Manifest file not available" } });
}
var (digest, size) = ComputeFileHash(fullPath);
// Quote the ETag so HttpClient parses it into response.Headers.ETag.
context.Response.Headers.ETag = $"\"{digest}\"";
context.Response.ContentType = "application/json";
context.Response.ContentLength = size;
return Results.File(fullPath, "application/json");
}).WithName("GetVexEvidenceLockerManifestFile");
// GET /evidence/vex/locker/{bundleId}/evidence/file
app.MapGet("/evidence/vex/locker/{bundleId}/evidence/file", async (
HttpContext context,
string bundleId,
[FromQuery] string? generation,
IOptions<VexStorageOptions> storageOptions,
IOptions<AirgapOptions> airgapOptions,
[FromServices] IAirgapImportStore airgapImportStore,
CancellationToken cancellationToken) =>
{
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, out var tenant, out var tenantError))
{
return tenantError;
}
var root = airgapOptions.Value.LockerRootPath;
if (string.IsNullOrWhiteSpace(root))
{
return Results.NotFound(new { error = new { code = "ERR_LOCKER_ROOT", message = "LockerRootPath is not configured" } });
}
var record = await airgapImportStore.FindByBundleIdAsync(tenant, bundleId.Trim(), generation?.Trim(), cancellationToken)
.ConfigureAwait(false);
if (record is null)
{
return Results.NotFound(new { error = new { code = "ERR_NOT_FOUND", message = "Evidence file not found" } });
}
if (!TryResolveLockerFile(root, record.EvidenceLockerPath, out var fullPath))
{
return Results.NotFound(new { error = new { code = "ERR_EVIDENCE_FILE", message = "Evidence file not available" } });
}
var (digest, size) = ComputeFileHash(fullPath);
// Quote the ETag so HttpClient parses it into response.Headers.ETag.
context.Response.Headers.ETag = $"\"{digest}\"";
context.Response.ContentType = "application/x-ndjson";
context.Response.ContentLength = size;
return Results.File(fullPath, "application/x-ndjson");
}).WithName("GetVexEvidenceLockerEvidenceFile");
}
private static void TryHashFile(string root, string relativePath, IVexHashingService hashingService, out string? digest, out long? size)
{
digest = null;
size = null;
try
{
if (string.IsNullOrWhiteSpace(relativePath))
{
return;
}
if (!TryResolveLockerFile(root, relativePath, out var fullPath))
{
return;
}
var data = File.ReadAllBytes(fullPath);
digest = hashingService.ComputeHash(data, "sha256");
size = data.LongLength;
}
catch
{
// Ignore I/O errors and continue with stored metadata
}
}
private static bool TryResolveLockerFile(string root, string relativePath, out string fullPath)
{
fullPath = string.Empty;
if (string.IsNullOrWhiteSpace(root) || string.IsNullOrWhiteSpace(relativePath))
{
return false;
}
var rootFull = Path.GetFullPath(root);
var candidate = Path.GetFullPath(Path.Combine(rootFull, relativePath));
if (!candidate.StartsWith(rootFull, StringComparison.OrdinalIgnoreCase))
{
return false;
}
if (!File.Exists(candidate))
{
return false;
}
fullPath = candidate;
return true;
}
private static (string Digest, long SizeBytes) ComputeFileHash(string path)
{
using var stream = File.OpenRead(path);
using var sha = SHA256.Create();
var hashBytes = sha.ComputeHash(stream);
var digest = "sha256:" + Convert.ToHexString(hashBytes).ToLowerInvariant();
var size = new FileInfo(path).Length;
return (digest, size);
}
private static bool TryResolveTenant(HttpContext context, VexStorageOptions options, out string tenant, out IResult? problem)
{
tenant = options.DefaultTenant;
problem = null;
if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerValues) && headerValues.Count > 0)
{
var requestedTenant = headerValues[0]?.Trim();
if (string.IsNullOrEmpty(requestedTenant))
{
problem = Results.BadRequest(new { error = new { code = "ERR_TENANT", message = "X-Stella-Tenant header must not be empty" } });
return false;
}
if (!string.Equals(requestedTenant, options.DefaultTenant, StringComparison.OrdinalIgnoreCase))
{
problem = Results.Json(
new { error = new { code = "ERR_TENANT_FORBIDDEN", message = $"Tenant '{requestedTenant}' is not allowed" } },
statusCode: StatusCodes.Status403Forbidden);
return false;
}
tenant = requestedTenant;
}
return true;
}
private static bool TryDecodeCursor(string cursor, out DateTime timestamp, out string id)
{
timestamp = default;
id = string.Empty;
try
{
var payload = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(cursor));
var parts = payload.Split('|');
if (parts.Length != 2)
{
return false;
}
if (!DateTimeOffset.TryParse(parts[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed))
{
return false;
}
timestamp = parsed.UtcDateTime;
id = parts[1];
return true;
}
catch
{
return false;
}
}
private static string EncodeCursor(DateTime timestamp, string id)
{
var payload = FormattableString.Invariant($"{timestamp:O}|{id}");
return Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(payload));
}
private static string EscapeRegex(string input)
{
// Escape special regex characters for safe use in MongoDB regex
return System.Text.RegularExpressions.Regex.Escape(input);
}
private static string? DeriveTrustTier(string? issuerId)
{
if (string.IsNullOrWhiteSpace(issuerId))
{
return null;
}
var lowerIssuerId = issuerId.ToLowerInvariant();
if (lowerIssuerId.Contains("vendor") || lowerIssuerId.Contains("upstream"))
{
return "vendor";
}
if (lowerIssuerId.Contains("distro") || lowerIssuerId.Contains("rhel") ||
lowerIssuerId.Contains("ubuntu") || lowerIssuerId.Contains("debian"))
{
return "distro-trusted";
}
if (lowerIssuerId.Contains("community") || lowerIssuerId.Contains("oss"))
{
return "community";
}
return "other";
chunkTelemetry.RecordIngested(tenant, null, "unavailable", "storage-migration", 0, 0, 0);
return Results.Problem(
detail: "Evidence chunk streaming is temporarily unavailable during Postgres migration (Mongo/BSON removed).",
statusCode: StatusCodes.Status503ServiceUnavailable,
title: "Service unavailable");
}).WithName("GetVexEvidenceChunks");
}
}

40
src/Excititor/StellaOps.Excititor.WebService/Program.cs
View File

@@ -48,6 +48,9 @@ services.AddOptions<VexStorageOptions>()
.ValidateOnStart();
services.AddExcititorPostgresStorage(configuration);
services.TryAddSingleton<IVexProviderStore, InMemoryVexProviderStore>();
services.TryAddSingleton<IVexConnectorStateRepository, InMemoryVexConnectorStateRepository>();
services.TryAddSingleton<IVexClaimStore, InMemoryVexClaimStore>();
services.AddCsafNormalizer();
services.AddCycloneDxNormalizer();
services.AddOpenVexNormalizer();
@@ -146,13 +149,12 @@ app.UseObservabilityHeaders();
app.MapGet("/excititor/status", async (HttpContext context,
IEnumerable<IVexArtifactStore> artifactStores,
IOptions<VexStorageOptions> mongoOptions,
IOptions<VexStorageOptions> storageOptions,
TimeProvider timeProvider) =>
{
var payload = new StatusResponse(
timeProvider.GetUtcNow(),
mongoOptions.Value.RawBucketName,
mongoOptions.Value.GridFsInlineThresholdBytes,
storageOptions.Value.InlineThresholdBytes,
artifactStores.Select(store => store.GetType().Name).ToArray());
context.Response.ContentType = "application/json";
@@ -210,19 +212,18 @@ app.MapGet("/openapi/excititor.json", () =>
{
schema = new { @ref = "#/components/schemas/StatusResponse" },
examples = new Dictionary<string, object>
{
["example"] = new
{
value = new
{
timeUtc = "2025-11-24T00:00:00Z",
mongoBucket = "vex-raw",
gridFsInlineThresholdBytes = 1048576,
artifactStores = new[] { "S3ArtifactStore", "OfflineBundleArtifactStore" }
}
}
}
}
{
["example"] = new
{
value = new
{
timeUtc = "2025-11-24T00:00:00Z",
inlineThreshold = 1048576,
artifactStores = new[] { "S3ArtifactStore", "OfflineBundleArtifactStore" }
}
}
}
}
}
}
}
@@ -892,12 +893,11 @@ app.MapGet("/openapi/excititor.json", () =>
["StatusResponse"] = new
{
type = "object",
required = new[] { "timeUtc", "mongoBucket", "artifactStores" },
required = new[] { "timeUtc", "artifactStores", "inlineThreshold" },
properties = new Dictionary<string, object>
{
["timeUtc"] = new { type = "string", format = "date-time" },
["mongoBucket"] = new { type = "string" },
["gridFsInlineThresholdBytes"] = new { type = "integer", format = "int64" },
["inlineThreshold"] = new { type = "integer", format = "int64" },
["artifactStores"] = new { type = "array", items = new { type = "string" } }
}
},
@@ -2270,7 +2270,7 @@ internal sealed record ExcititorTimelineEvent(
public partial class Program;
internal sealed record StatusResponse(DateTimeOffset UtcNow, string MongoBucket, int InlineThreshold, string[] ArtifactStores);
internal sealed record StatusResponse(DateTimeOffset UtcNow, int InlineThreshold, string[] ArtifactStores);
internal sealed record VexStatementIngestRequest(IReadOnlyList<VexStatementEntry> Statements);

233
src/Excititor/StellaOps.Excititor.WebService/Services/ExcititorHealthService.cs
View File

@@ -1,48 +1,49 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using MongoDB.Bson;
using MongoDB.Driver;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
using StellaOps.Excititor.Core.Observations;
using StellaOps.Excititor.WebService.Options;
namespace StellaOps.Excititor.WebService.Services;
internal sealed class ExcititorHealthService
{
private const string RetrievedAtField = "RetrievedAt";
private const string MetadataField = "Metadata";
private const string CalculatedAtField = "CalculatedAt";
private const string ConflictsField = "Conflicts";
private const string ConflictStatusField = "Status";
private readonly IMongoDatabase _database;
private readonly IVexRawStore _rawStore;
private readonly IVexLinksetStore _linksetStore;
private readonly IVexProviderStore _providerStore;
private readonly IVexConnectorStateRepository _stateRepository;
private readonly IReadOnlyDictionary<string, VexConnectorDescriptor> _connectors;
private readonly TimeProvider _timeProvider;
private readonly ExcititorObservabilityOptions _options;
private readonly ILogger<ExcititorHealthService> _logger;
private readonly string _defaultTenant;
public ExcititorHealthService(
IMongoDatabase database,
IVexRawStore rawStore,
IVexLinksetStore linksetStore,
IVexProviderStore providerStore,
IVexConnectorStateRepository stateRepository,
IEnumerable<IVexConnector> connectors,
TimeProvider timeProvider,
IOptions<ExcititorObservabilityOptions> options,
IOptions<VexStorageOptions> storageOptions,
ILogger<ExcititorHealthService> logger)
{
_database = database ?? throw new ArgumentNullException(nameof(database));
_rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore));
_linksetStore = linksetStore ?? throw new ArgumentNullException(nameof(linksetStore));
_providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore));
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
_timeProvider = timeProvider ?? TimeProvider.System;
_options = options?.Value ?? new ExcititorObservabilityOptions();
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
var storage = storageOptions?.Value ?? new VexStorageOptions();
_defaultTenant = string.IsNullOrWhiteSpace(storage.DefaultTenant)
? "default"
: storage.DefaultTenant.Trim();
if (connectors is null)
{
@@ -158,7 +159,7 @@ internal sealed class ExcititorHealthService
private LinkHealthSection BuildLinkSection(DateTimeOffset now, LinkSnapshot snapshot)
{
TimeSpan? lag = null;
if (snapshot.LastConsensusAt is { } calculatedAt)
if (snapshot.LastUpdatedAt is { } calculatedAt)
{
lag = now - calculatedAt;
if (lag < TimeSpan.Zero)
@@ -174,7 +175,7 @@ internal sealed class ExcititorHealthService
return new LinkHealthSection(
status,
snapshot.LastConsensusAt,
snapshot.LastUpdatedAt,
lag?.TotalSeconds,
snapshot.TotalDocuments,
snapshot.DocumentsWithConflicts);
@@ -271,47 +272,36 @@ internal sealed class ExcititorHealthService
var window = _options.GetPositive(_options.SignatureWindow, TimeSpan.FromHours(12));
var windowStart = now - window;
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Raw);
var filter = Builders<BsonDocument>.Filter.Gte(RetrievedAtField, windowStart.UtcDateTime);
var projection = Builders<BsonDocument>.Projection
.Include(MetadataField)
.Include(RetrievedAtField);
List<BsonDocument> documents;
try
{
documents = await collection
.Find(filter)
.Project(projection)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load signature window metrics.");
documents = new List<BsonDocument>();
}
var page = await _rawStore.QueryAsync(
new VexRawQuery(
_defaultTenant,
Array.Empty<string>(),
Array.Empty<string>(),
Array.Empty<VexDocumentFormat>(),
windowStart,
until: null,
Cursor: null,
Limit: 500),
cancellationToken).ConfigureAwait(false);
var evaluated = 0;
var withSignatures = 0;
var verified = 0;
foreach (var document in documents)
foreach (var document in page.Items)
{
evaluated++;
if (!document.TryGetValue(MetadataField, out var metadataValue) ||
metadataValue is not BsonDocument metadata ||
metadata.ElementCount == 0)
{
continue;
}
if (TryGetBoolean(metadata, "signature.present", out var present) && present)
var metadata = document.Metadata;
if (metadata.TryGetValue("signature.present", out var presentValue) &&
bool.TryParse(presentValue, out var present) &&
present)
{
withSignatures++;
}
if (TryGetBoolean(metadata, "signature.verified", out var verifiedFlag) && verifiedFlag)
if (metadata.TryGetValue("signature.verified", out var verifiedValue) &&
bool.TryParse(verifiedValue, out var verifiedFlag) &&
verifiedFlag)
{
verified++;
}
@@ -322,80 +312,43 @@ internal sealed class ExcititorHealthService
private async Task<LinkSnapshot> LoadLinkSnapshotAsync(CancellationToken cancellationToken)
{
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
BsonDocument? latest = null;
try
{
latest = await collection
.Find(Builders<BsonDocument>.Filter.Empty)
.Sort(Builders<BsonDocument>.Sort.Descending(CalculatedAtField))
.Project(Builders<BsonDocument>.Projection.Include(CalculatedAtField))
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to read latest consensus document.");
}
DateTimeOffset? lastConsensusAt = null;
if (latest is not null &&
latest.TryGetValue(CalculatedAtField, out var dateValue))
{
var utc = TryReadDateTime(dateValue);
if (utc is not null)
{
lastConsensusAt = new DateTimeOffset(utc.Value, TimeSpan.Zero);
}
}
long totalDocuments = 0;
long conflictDocuments = 0;
DateTimeOffset? lastUpdated = null;
try
{
totalDocuments = await collection.EstimatedDocumentCountAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
conflictDocuments = await collection.CountDocumentsAsync(
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"),
cancellationToken: cancellationToken)
.ConfigureAwait(false);
totalDocuments = await _linksetStore.CountAsync(_defaultTenant, cancellationToken).ConfigureAwait(false);
conflictDocuments = await _linksetStore.CountWithConflictsAsync(_defaultTenant, cancellationToken).ConfigureAwait(false);
var conflictSample = await _linksetStore.FindWithConflictsAsync(_defaultTenant, 1, cancellationToken).ConfigureAwait(false);
if (conflictSample.Count > 0)
{
lastUpdated = conflictSample[0].UpdatedAt;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to compute consensus counts.");
_logger.LogWarning(ex, "Failed to compute linkset counts.");
}
return new LinkSnapshot(lastConsensusAt, totalDocuments, conflictDocuments);
return new LinkSnapshot(lastUpdated, totalDocuments, conflictDocuments);
}
private async Task<ConflictSnapshot> LoadConflictSnapshotAsync(DateTimeOffset now, CancellationToken cancellationToken)
{
var window = _options.GetPositive(_options.ConflictTrendWindow, TimeSpan.FromHours(24));
var windowStart = now - window;
var collection = _database.GetCollection<BsonDocument>(VexMongoCollectionNames.Consensus);
var filter = Builders<BsonDocument>.Filter.And(
Builders<BsonDocument>.Filter.Gte(CalculatedAtField, windowStart.UtcDateTime),
Builders<BsonDocument>.Filter.Exists($"{ConflictsField}.0"));
var projection = Builders<BsonDocument>.Projection
.Include(CalculatedAtField)
.Include(ConflictsField);
List<BsonDocument> documents;
IReadOnlyList<VexLinkset> linksets;
try
{
documents = await collection
.Find(filter)
.Project(projection)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
// Sample conflicted linksets (ordered by updated_at DESC in Postgres implementation)
linksets = await _linksetStore.FindWithConflictsAsync(_defaultTenant, 500, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to load conflict trend window.");
documents = new List<BsonDocument>();
linksets = Array.Empty<VexLinkset>();
}
var byStatus = new Dictionary<string, long>(StringComparer.OrdinalIgnoreCase);
@@ -405,47 +358,31 @@ internal sealed class ExcititorHealthService
var bucketMinutes = Math.Max(1, _options.ConflictTrendBucketMinutes);
var bucketTicks = TimeSpan.FromMinutes(bucketMinutes).Ticks;
foreach (var doc in documents)
foreach (var linkset in linksets)
{
if (!doc.TryGetValue(ConflictsField, out var conflictsValue) ||
conflictsValue is not BsonArray conflicts ||
conflicts.Count == 0)
if (linkset.Disagreements.Count == 0)
{
continue;
}
docsWithConflicts++;
totalConflicts += conflicts.Count;
totalConflicts += linkset.Disagreements.Count;
foreach (var conflictValue in conflicts.OfType<BsonDocument>())
foreach (var disagreement in linkset.Disagreements)
{
var status = conflictValue.TryGetValue(ConflictStatusField, out var statusValue) && statusValue.IsString
? statusValue.AsString
: "unknown";
if (string.IsNullOrWhiteSpace(status))
{
status = "unknown";
}
var status = string.IsNullOrWhiteSpace(disagreement.Status)
? "unknown"
: disagreement.Status;
byStatus[status] = byStatus.TryGetValue(status, out var current)
? current + 1
: 1;
}
if (doc.TryGetValue(CalculatedAtField, out var calculatedValue))
{
var utc = TryReadDateTime(calculatedValue);
if (utc is null)
{
continue;
}
var alignedTicks = AlignTicks(utc.Value, bucketTicks);
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var current)
? current + conflicts.Count
: conflicts.Count;
}
var alignedTicks = AlignTicks(linkset.UpdatedAt.UtcDateTime, bucketTicks);
timeline[alignedTicks] = timeline.TryGetValue(alignedTicks, out var currentCount)
? currentCount + linkset.Disagreements.Count
: linkset.Disagreements.Count;
}
var trend = timeline
@@ -541,54 +478,6 @@ internal sealed class ExcititorHealthService
return ticks - (ticks % bucketTicks);
}
private static DateTime? TryReadDateTime(BsonValue value)
{
if (value is null)
{
return null;
}
if (value.IsBsonDateTime)
{
return value.AsBsonDateTime.ToUniversalTime();
}
if (value.IsString &&
DateTime.TryParse(
value.AsString,
CultureInfo.InvariantCulture,
DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal,
out var parsed))
{
return DateTime.SpecifyKind(parsed, DateTimeKind.Utc);
}
return null;
}
private static bool TryGetBoolean(BsonDocument document, string key, out bool value)
{
value = default;
if (!document.TryGetValue(key, out var bsonValue))
{
return false;
}
if (bsonValue.IsBoolean)
{
value = bsonValue.AsBoolean;
return true;
}
if (bsonValue.IsString && bool.TryParse(bsonValue.AsString, out var parsed))
{
value = parsed;
return true;
}
return false;
}
private static VexConnectorDescriptor DescribeConnector(IVexConnector connector)
=> connector switch
{
@@ -596,7 +485,7 @@ internal sealed class ExcititorHealthService
_ => new VexConnectorDescriptor(connector.Id, connector.Kind, connector.Id)
};
private sealed record LinkSnapshot(DateTimeOffset? LastConsensusAt, long TotalDocuments, long DocumentsWithConflicts);
private sealed record LinkSnapshot(DateTimeOffset? LastUpdatedAt, long TotalDocuments, long DocumentsWithConflicts);
private sealed record ConflictSnapshot(
DateTimeOffset WindowStart,

32
src/Excititor/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs
View File

@@ -5,7 +5,6 @@ using System.Globalization;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using StellaOps.Excititor.Connectors.Abstractions;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Storage;
@@ -151,7 +150,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
foreach (var handle in handles)
{
var result = await ExecuteRunAsync(runId, handle, since, options.Force, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, options.Force, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
@@ -174,8 +173,8 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
foreach (var handle in handles)
{
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, cancellationToken).ConfigureAwait(false);
results.Add(result);
}
@@ -201,14 +200,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
{
try
{
var state = await _stateRepository.GetAsync(handle.Descriptor.Id, cancellationToken, session).ConfigureAwait(false);
var state = await _stateRepository.GetAsync(handle.Descriptor.Id, cancellationToken).ConfigureAwait(false);
var lastUpdated = state?.LastUpdated;
var stale = threshold.HasValue && (lastUpdated is null || lastUpdated < threshold.Value);
if (stale || state is null)
{
var since = stale ? threshold : lastUpdated;
var result = await ExecuteRunAsync(runId, handle, since, force: false, session, cancellationToken).ConfigureAwait(false);
var result = await ExecuteRunAsync(runId, handle, since, force: false, cancellationToken).ConfigureAwait(false);
results.Add(new ReconcileProviderResult(
handle.Descriptor.Id,
result.Status,
@@ -271,14 +270,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
private async Task EnsureProviderRegistrationAsync(VexConnectorDescriptor descriptor, CancellationToken cancellationToken)
{
var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false);
var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken).ConfigureAwait(false);
if (existing is not null)
{
return;
}
var provider = new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind);
await _providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false);
await _providerStore.SaveAsync(provider, cancellationToken).ConfigureAwait(false);
}
private async Task<ProviderRunResult> ExecuteRunAsync(
@@ -286,7 +285,6 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
ConnectorHandle handle,
DateTimeOffset? since,
bool force,
IClientSessionHandle session,
CancellationToken cancellationToken)
{
var providerId = handle.Descriptor.Id;
@@ -304,15 +302,15 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
try
{
await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false);
await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false);
await EnsureProviderRegistrationAsync(handle.Descriptor, cancellationToken).ConfigureAwait(false);
if (force)
{
var resetState = new VexConnectorState(providerId, null, ImmutableArray<string>.Empty);
await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false);
await _stateRepository.SaveAsync(resetState, cancellationToken).ConfigureAwait(false);
}
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var stateBeforeRun = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
var resumeTokens = stateBeforeRun?.ResumeTokens ?? ImmutableDictionary<string, string>.Empty;
var context = new VexConnectorContext(
@@ -337,13 +335,13 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0)
{
claims += batch.Claims.Length;
await _claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false);
await _claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
}
}
stopwatch.Stop();
var completedAt = _timeProvider.GetUtcNow();
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var stateAfterRun = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
var checkpoint = stateAfterRun?.DocumentDigests.IsDefaultOrEmpty == false
? stateAfterRun.DocumentDigests[^1]
@@ -413,7 +411,7 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
}
}
private async Task<DateTimeOffset?> ResolveResumeSinceAsync(string providerId, string? checkpoint, IClientSessionHandle session, CancellationToken cancellationToken)
private async Task<DateTimeOffset?> ResolveResumeSinceAsync(string providerId, string? checkpoint, CancellationToken cancellationToken)
{
if (!string.IsNullOrWhiteSpace(checkpoint))
{
@@ -427,14 +425,14 @@ internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator
}
var digest = checkpoint.Trim();
var document = await _rawStore.FindByDigestAsync(digest, cancellationToken, session).ConfigureAwait(false);
var document = await _rawStore.FindByDigestAsync(digest, cancellationToken).ConfigureAwait(false);
if (document is not null)
{
return document.RetrievedAt;
}
}
var state = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false);
var state = await _stateRepository.GetAsync(providerId, cancellationToken).ConfigureAwait(false);
return state?.LastUpdated;
}

2
src/Excititor/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj
View File

@@ -17,7 +17,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Storage.Postgres/StellaOps.Excititor.Storage.Postgres.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj" />
<ProjectReference Include="../__Libraries/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj" />