Add new features and tests for AirGap and Time modules
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Introduced `SbomService` tasks documentation.
- Updated `StellaOps.sln` to include new projects: `StellaOps.AirGap.Time` and `StellaOps.AirGap.Importer`.
- Added unit tests for `BundleImportPlanner`, `DsseVerifier`, `ImportValidator`, and other components in the `StellaOps.AirGap.Importer.Tests` namespace.
- Implemented `InMemoryBundleRepositories` for testing bundle catalog and item repositories.
- Created `MerkleRootCalculator`, `RootRotationPolicy`, and `TufMetadataValidator` tests.
- Developed `StalenessCalculator` and `TimeAnchorLoader` tests in the `StellaOps.AirGap.Time.Tests` namespace.
- Added `fetch-sbomservice-deps.sh` script for offline dependency fetching.
This commit is contained in:
master
2025-11-20 23:29:54 +02:00
parent 65b1599229
commit 79b8e53441
182 changed files with 6660 additions and 1242 deletions

View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Collections.Immutable;
using System.Globalization;
using System.Diagnostics;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Authentication;
@@ -13,16 +14,16 @@ using Microsoft.Extensions.Options;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Primitives;
using StellaOps.Excititor.Attestation.Verification;
using StellaOps.Excititor.Attestation.Extensions;
using StellaOps.Excititor.Attestation;
using StellaOps.Excititor.Attestation.Transparency;
using StellaOps.Excititor.ArtifactStores.S3.Extensions;
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export;
using StellaOps.Excititor.Formats.CSAF;
using StellaOps.Excititor.Formats.CycloneDX;
using StellaOps.Excititor.Formats.OpenVEX;
using StellaOps.Excititor.Attestation.Extensions;
using StellaOps.Excititor.Attestation;
using StellaOps.Excititor.Attestation.Transparency;
using StellaOps.Excititor.ArtifactStores.S3.Extensions;
using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection;
using StellaOps.Excititor.Core;
using StellaOps.Excititor.Export;
using StellaOps.Excititor.Formats.CSAF;
using StellaOps.Excititor.Formats.CycloneDX;
using StellaOps.Excititor.Formats.OpenVEX;
using StellaOps.Excititor.Policy;
using StellaOps.Excititor.Storage.Mongo;
using StellaOps.Excititor.WebService.Endpoints;
@@ -34,14 +35,14 @@ using StellaOps.Excititor.WebService.Contracts;
using StellaOps.Excititor.WebService.Telemetry;
using MongoDB.Driver;
using MongoDB.Bson;
var builder = WebApplication.CreateBuilder(args);
var configuration = builder.Configuration;
var services = builder.Services;
services.AddOptions<VexMongoStorageOptions>()
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
.ValidateOnStart();
services.AddOptions<VexMongoStorageOptions>()
.Bind(configuration.GetSection("Excititor:Storage:Mongo"))
.ValidateOnStart();
services.AddExcititorMongoStorage();
services.AddCsafNormalizer();
services.AddCycloneDxNormalizer();
@@ -60,6 +61,8 @@ services.Configure<VexAttestationClientOptions>(configuration.GetSection("Exciti
services.Configure<VexAttestationVerificationOptions>(configuration.GetSection("Excititor:Attestation:Verification"));
services.AddVexPolicy();
services.AddSingleton<IVexEvidenceChunkService, VexEvidenceChunkService>();
services.AddSingleton<ChunkTelemetry>();
services.AddSingleton<ChunkTelemetry>();
services.AddRedHatCsafConnector();
services.Configure<MirrorDistributionOptions>(configuration.GetSection(MirrorDistributionOptions.SectionName));
services.AddSingleton<MirrorRateLimiter>();
@@ -67,47 +70,47 @@ services.TryAddSingleton(TimeProvider.System);
services.AddSingleton<IVexObservationProjectionService, VexObservationProjectionService>();
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
services.AddScoped<IVexObservationLookup, MongoVexObservationLookup>();
var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor");
if (rekorSection.Exists())
{
services.AddVexRekorClient(opts => rekorSection.Bind(opts));
}
var fileSystemSection = configuration.GetSection("Excititor:Artifacts:FileSystem");
if (fileSystemSection.Exists())
{
services.AddVexFileSystemArtifactStore(opts => fileSystemSection.Bind(opts));
}
else
{
services.AddVexFileSystemArtifactStore(_ => { });
}
var s3Section = configuration.GetSection("Excititor:Artifacts:S3");
if (s3Section.Exists())
{
services.AddVexS3ArtifactClient(opts => s3Section.GetSection("Client").Bind(opts));
services.AddSingleton<IVexArtifactStore, S3ArtifactStore>(provider =>
{
var options = new S3ArtifactStoreOptions();
s3Section.GetSection("Store").Bind(options);
return new S3ArtifactStore(
provider.GetRequiredService<IS3ArtifactClient>(),
Microsoft.Extensions.Options.Options.Create(options),
provider.GetRequiredService<Microsoft.Extensions.Logging.ILogger<S3ArtifactStore>>());
});
}
var offlineSection = configuration.GetSection("Excititor:Artifacts:OfflineBundle");
if (offlineSection.Exists())
{
services.AddVexOfflineBundleArtifactStore(opts => offlineSection.Bind(opts));
}
services.AddEndpointsApiExplorer();
services.AddHealthChecks();
services.AddSingleton(TimeProvider.System);
var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor");
if (rekorSection.Exists())
{
services.AddVexRekorClient(opts => rekorSection.Bind(opts));
}
var fileSystemSection = configuration.GetSection("Excititor:Artifacts:FileSystem");
if (fileSystemSection.Exists())
{
services.AddVexFileSystemArtifactStore(opts => fileSystemSection.Bind(opts));
}
else
{
services.AddVexFileSystemArtifactStore(_ => { });
}
var s3Section = configuration.GetSection("Excititor:Artifacts:S3");
if (s3Section.Exists())
{
services.AddVexS3ArtifactClient(opts => s3Section.GetSection("Client").Bind(opts));
services.AddSingleton<IVexArtifactStore, S3ArtifactStore>(provider =>
{
var options = new S3ArtifactStoreOptions();
s3Section.GetSection("Store").Bind(options);
return new S3ArtifactStore(
provider.GetRequiredService<IS3ArtifactClient>(),
Microsoft.Extensions.Options.Options.Create(options),
provider.GetRequiredService<Microsoft.Extensions.Logging.ILogger<S3ArtifactStore>>());
});
}
var offlineSection = configuration.GetSection("Excititor:Artifacts:OfflineBundle");
if (offlineSection.Exists())
{
services.AddVexOfflineBundleArtifactStore(opts => offlineSection.Bind(opts));
}
services.AddEndpointsApiExplorer();
services.AddHealthChecks();
services.AddSingleton(TimeProvider.System);
services.AddMemoryCache();
services.AddAuthentication();
services.AddAuthorization();
@@ -115,70 +118,134 @@ services.AddAuthorization();
builder.ConfigureExcititorTelemetry();
var app = builder.Build();
app.UseAuthentication();
app.UseAuthorization();
app.UseObservabilityHeaders();
app.MapGet("/excititor/status", async (HttpContext context,
IEnumerable<IVexArtifactStore> artifactStores,
IOptions<VexMongoStorageOptions> mongoOptions,
TimeProvider timeProvider) =>
{
var payload = new StatusResponse(
timeProvider.GetUtcNow(),
mongoOptions.Value.RawBucketName,
mongoOptions.Value.GridFsInlineThresholdBytes,
artifactStores.Select(store => store.GetType().Name).ToArray());
context.Response.ContentType = "application/json";
await System.Text.Json.JsonSerializer.SerializeAsync(context.Response.Body, payload);
});
app.MapHealthChecks("/excititor/health");
app.MapPost("/excititor/statements", async (
VexStatementIngestRequest request,
IVexClaimStore claimStore,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
if (request?.Statements is null || request.Statements.Count == 0)
{
return Results.BadRequest("At least one statement must be provided.");
}
var claims = request.Statements.Select(statement => statement.ToDomainClaim());
await claimStore.AppendAsync(claims, timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
return Results.Accepted();
});
app.MapGet("/excititor/statements/{vulnerabilityId}/{productKey}", async (
string vulnerabilityId,
string productKey,
DateTimeOffset? since,
IVexClaimStore claimStore,
CancellationToken cancellationToken) =>
{
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
{
return Results.BadRequest("vulnerabilityId and productKey are required.");
}
var claims = await claimStore.FindAsync(vulnerabilityId.Trim(), productKey.Trim(), since, cancellationToken).ConfigureAwait(false);
return Results.Ok(claims);
});
app.MapGet("/excititor/status", async (HttpContext context,
IEnumerable<IVexArtifactStore> artifactStores,
IOptions<VexMongoStorageOptions> mongoOptions,
TimeProvider timeProvider) =>
{
var payload = new StatusResponse(
timeProvider.GetUtcNow(),
mongoOptions.Value.RawBucketName,
mongoOptions.Value.GridFsInlineThresholdBytes,
artifactStores.Select(store => store.GetType().Name).ToArray());
context.Response.ContentType = "application/json";
await System.Text.Json.JsonSerializer.SerializeAsync(context.Response.Body, payload);
});
app.MapHealthChecks("/excititor/health");
app.MapPost("/v1/attestations/verify", async (
[FromServices] IVexAttestationClient attestationClient,
[FromBody] AttestationVerifyRequest request,
CancellationToken cancellationToken) =>
{
if (request is null)
{
return Results.BadRequest("Request body is required.");
}
if (string.IsNullOrWhiteSpace(request.ExportId) ||
string.IsNullOrWhiteSpace(request.QuerySignature) ||
string.IsNullOrWhiteSpace(request.ArtifactDigest) ||
string.IsNullOrWhiteSpace(request.Format) ||
string.IsNullOrWhiteSpace(request.Envelope) ||
string.IsNullOrWhiteSpace(request.Attestation?.EnvelopeDigest))
{
return Results.BadRequest("Missing required fields.");
}
if (!Enum.TryParse<VexExportFormat>(request.Format, ignoreCase: true, out var format))
{
return Results.BadRequest("Unknown export format.");
}
var attestationRequest = new VexAttestationRequest(
request.ExportId.Trim(),
new VexQuerySignature(request.QuerySignature.Trim()),
new VexContentAddress(request.ArtifactDigest.Trim()),
format,
request.CreatedAt,
request.SourceProviders?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
request.Metadata?.ToImmutableDictionary(StringComparer.Ordinal) ?? ImmutableDictionary<string, string>.Empty);
var rekor = request.Attestation?.Rekor is null
? null
: new VexRekorReference(
request.Attestation.Rekor.ApiVersion ?? "0.2",
request.Attestation.Rekor.Location,
request.Attestation.Rekor.LogIndex,
request.Attestation.Rekor.InclusionProofUrl);
var attestationMetadata = new VexAttestationMetadata(
request.Attestation?.PredicateType ?? string.Empty,
rekor,
request.Attestation!.EnvelopeDigest,
request.Attestation.SignedAt);
var verificationRequest = new VexAttestationVerificationRequest(
attestationRequest,
attestationMetadata,
request.Envelope,
request.IsReverify);
var verification = await attestationClient.VerifyAsync(verificationRequest, cancellationToken).ConfigureAwait(false);
var response = new AttestationVerifyResponse(
verification.IsValid,
new Dictionary<string, string>(verification.Diagnostics, StringComparer.Ordinal));
return Results.Ok(response);
});
app.MapPost("/excititor/statements"
, async (
VexStatementIngestRequest request,
IVexClaimStore claimStore,
TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
if (request?.Statements is null || request.Statements.Count == 0)
{
return Results.BadRequest("At least one statement must be provided.");
}
var claims = request.Statements.Select(statement => statement.ToDomainClaim());
await claimStore.AppendAsync(claims, timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false);
return Results.Accepted();
});
app.MapGet("/excititor/statements/{vulnerabilityId}/{productKey}", async (
string vulnerabilityId,
string productKey,
DateTimeOffset? since,
IVexClaimStore claimStore,
CancellationToken cancellationToken) =>
{
if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey))
{
return Results.BadRequest("vulnerabilityId and productKey are required.");
}
var claims = await claimStore.FindAsync(vulnerabilityId.Trim(), productKey.Trim(), since, cancellationToken).ConfigureAwait(false);
return Results.Ok(claims);
});
app.MapPost("/excititor/admin/backfill-statements", async (
VexStatementBackfillRequest? request,
VexStatementBackfillService backfillService,
CancellationToken cancellationToken) =>
{
request ??= new VexStatementBackfillRequest();
var result = await backfillService.RunAsync(request, cancellationToken).ConfigureAwait(false);
var message = FormattableString.Invariant(
$"Backfill completed: evaluated {result.DocumentsEvaluated}, backfilled {result.DocumentsBackfilled}, claims written {result.ClaimsWritten}, skipped {result.SkippedExisting}, failures {result.NormalizationFailures}.");
request ??= new VexStatementBackfillRequest();
var result = await backfillService.RunAsync(request, cancellationToken).ConfigureAwait(false);
var message = FormattableString.Invariant(
$"Backfill completed: evaluated {result.DocumentsEvaluated}, backfilled {result.DocumentsBackfilled}, claims written {result.ClaimsWritten}, skipped {result.SkippedExisting}, failures {result.NormalizationFailures}.");
return Results.Ok(new
{
message,
@@ -742,17 +809,23 @@ app.MapGet("/v1/vex/evidence/chunks", async (
HttpContext context,
[FromServices] IVexEvidenceChunkService chunkService,
[FromServices] IOptions<VexMongoStorageOptions> storageOptions,
[FromServices] ChunkTelemetry chunkTelemetry,
[FromServices] ILogger<VexEvidenceChunkRequest> logger,
[FromServices] TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
var start = Stopwatch.GetTimestamp();
var scopeResult = ScopeAuthorization.RequireScope(context, "vex.read");
if (scopeResult is not null)
{
chunkTelemetry.RecordIngested(null, null, "unauthorized", "missing-scope", 0, 0, 0);
return scopeResult;
}
if (!TryResolveTenant(context, storageOptions.Value, requireHeader: false, out var tenant, out var tenantError))
{
chunkTelemetry.RecordIngested(tenant?.TenantId, null, "rejected", "tenant-invalid", 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
return tenantError;
}
@@ -785,11 +858,13 @@ app.MapGet("/v1/vex/evidence/chunks", async (
catch (OperationCanceledException)
{
EvidenceTelemetry.RecordChunkOutcome(tenant, "cancelled");
chunkTelemetry.RecordIngested(tenant?.TenantId, request.ProviderFilter.Count > 0 ? string.Join(',', request.ProviderFilter) : null, "cancelled", null, 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
return Results.StatusCode(StatusCodes.Status499ClientClosedRequest);
}
catch
{
EvidenceTelemetry.RecordChunkOutcome(tenant, "error");
chunkTelemetry.RecordIngested(tenant?.TenantId, request.ProviderFilter.Count > 0 ? string.Join(',', request.ProviderFilter) : null, "error", null, 0, 0, Stopwatch.GetElapsedTime(start).TotalMilliseconds);
throw;
}
@@ -814,13 +889,25 @@ app.MapGet("/v1/vex/evidence/chunks", async (
context.Response.ContentType = "application/x-ndjson";
var options = new JsonSerializerOptions(JsonSerializerDefaults.Web);
long payloadBytes = 0;
foreach (var chunk in result.Chunks)
{
var line = JsonSerializer.Serialize(chunk, options);
payloadBytes += Encoding.UTF8.GetByteCount(line) + 1;
await context.Response.WriteAsync(line, cancellationToken).ConfigureAwait(false);
await context.Response.WriteAsync("\n", cancellationToken).ConfigureAwait(false);
}
var elapsedMs = Stopwatch.GetElapsedTime(start).TotalMilliseconds;
chunkTelemetry.RecordIngested(
tenant?.TenantId,
request.ProviderFilter.Count > 0 ? string.Join(',', request.ProviderFilter) : null,
"success",
null,
result.TotalCount,
payloadBytes,
elapsedMs);
return Results.Empty;
});
@@ -969,107 +1056,107 @@ app.MapGet("/obs/excititor/health", async (
IngestEndpoints.MapIngestEndpoints(app);
ResolveEndpoint.MapResolveEndpoint(app);
MirrorEndpoints.MapMirrorEndpoints(app);
app.Run();
public partial class Program;
internal sealed record StatusResponse(DateTimeOffset UtcNow, string MongoBucket, int InlineThreshold, string[] ArtifactStores);
internal sealed record VexStatementIngestRequest(IReadOnlyList<VexStatementEntry> Statements);
internal sealed record VexStatementEntry(
string VulnerabilityId,
string ProviderId,
string ProductKey,
string? ProductName,
string? ProductVersion,
string? ProductPurl,
string? ProductCpe,
IReadOnlyList<string>? ComponentIdentifiers,
VexClaimStatus Status,
VexJustification? Justification,
string? Detail,
DateTimeOffset FirstSeen,
DateTimeOffset LastSeen,
VexDocumentFormat DocumentFormat,
string DocumentDigest,
string DocumentUri,
string? DocumentRevision,
VexSignatureMetadataRequest? Signature,
VexConfidenceRequest? Confidence,
VexSignalRequest? Signals,
IReadOnlyDictionary<string, string>? Metadata)
{
public VexClaim ToDomainClaim()
{
var product = new VexProduct(
ProductKey,
ProductName,
ProductVersion,
ProductPurl,
ProductCpe,
ComponentIdentifiers ?? Array.Empty<string>());
if (!Uri.TryCreate(DocumentUri, UriKind.Absolute, out var uri))
{
throw new InvalidOperationException($"DocumentUri '{DocumentUri}' is not a valid absolute URI.");
}
var document = new VexClaimDocument(
DocumentFormat,
DocumentDigest,
uri,
DocumentRevision,
Signature?.ToDomain());
var additionalMetadata = Metadata is null
? ImmutableDictionary<string, string>.Empty
: Metadata.ToImmutableDictionary(StringComparer.Ordinal);
return new VexClaim(
VulnerabilityId,
ProviderId,
product,
Status,
document,
FirstSeen,
LastSeen,
Justification,
Detail,
Confidence?.ToDomain(),
Signals?.ToDomain(),
additionalMetadata);
}
}
internal sealed record VexSignatureMetadataRequest(
string Type,
string? Subject,
string? Issuer,
string? KeyId,
DateTimeOffset? VerifiedAt,
string? TransparencyLogReference)
{
public VexSignatureMetadata ToDomain()
=> new(Type, Subject, Issuer, KeyId, VerifiedAt, TransparencyLogReference);
}
internal sealed record VexConfidenceRequest(string Level, double? Score, string? Method)
{
public VexConfidence ToDomain() => new(Level, Score, Method);
}
internal sealed record VexSignalRequest(VexSeveritySignalRequest? Severity, bool? Kev, double? Epss)
{
public VexSignalSnapshot ToDomain()
=> new(Severity?.ToDomain(), Kev, Epss);
}
internal sealed record VexSeveritySignalRequest(string Scheme, double? Score, string? Label, string? Vector)
{
public VexSeveritySignal ToDomain() => new(Scheme, Score, Label, Vector);
}
app.Run();
public partial class Program;
internal sealed record StatusResponse(DateTimeOffset UtcNow, string MongoBucket, int InlineThreshold, string[] ArtifactStores);
internal sealed record VexStatementIngestRequest(IReadOnlyList<VexStatementEntry> Statements);
internal sealed record VexStatementEntry(
string VulnerabilityId,
string ProviderId,
string ProductKey,
string? ProductName,
string? ProductVersion,
string? ProductPurl,
string? ProductCpe,
IReadOnlyList<string>? ComponentIdentifiers,
VexClaimStatus Status,
VexJustification? Justification,
string? Detail,
DateTimeOffset FirstSeen,
DateTimeOffset LastSeen,
VexDocumentFormat DocumentFormat,
string DocumentDigest,
string DocumentUri,
string? DocumentRevision,
VexSignatureMetadataRequest? Signature,
VexConfidenceRequest? Confidence,
VexSignalRequest? Signals,
IReadOnlyDictionary<string, string>? Metadata)
{
public VexClaim ToDomainClaim()
{
var product = new VexProduct(
ProductKey,
ProductName,
ProductVersion,
ProductPurl,
ProductCpe,
ComponentIdentifiers ?? Array.Empty<string>());
if (!Uri.TryCreate(DocumentUri, UriKind.Absolute, out var uri))
{
throw new InvalidOperationException($"DocumentUri '{DocumentUri}' is not a valid absolute URI.");
}
var document = new VexClaimDocument(
DocumentFormat,
DocumentDigest,
uri,
DocumentRevision,
Signature?.ToDomain());
var additionalMetadata = Metadata is null
? ImmutableDictionary<string, string>.Empty
: Metadata.ToImmutableDictionary(StringComparer.Ordinal);
return new VexClaim(
VulnerabilityId,
ProviderId,
product,
Status,
document,
FirstSeen,
LastSeen,
Justification,
Detail,
Confidence?.ToDomain(),
Signals?.ToDomain(),
additionalMetadata);
}
}
internal sealed record VexSignatureMetadataRequest(
string Type,
string? Subject,
string? Issuer,
string? KeyId,
DateTimeOffset? VerifiedAt,
string? TransparencyLogReference)
{
public VexSignatureMetadata ToDomain()
=> new(Type, Subject, Issuer, KeyId, VerifiedAt, TransparencyLogReference);
}
internal sealed record VexConfidenceRequest(string Level, double? Score, string? Method)
{
public VexConfidence ToDomain() => new(Level, Score, Method);
}
internal sealed record VexSignalRequest(VexSeveritySignalRequest? Severity, bool? Kev, double? Epss)
{
public VexSignalSnapshot ToDomain()
=> new(Severity?.ToDomain(), Kev, Epss);
}
internal sealed record VexSeveritySignalRequest(string Scheme, double? Score, string? Label, string? Vector)
{
public VexSeveritySignal ToDomain() => new(Scheme, Score, Label, Vector);
}
app.MapGet(
"/v1/vex/observations",
async (