feat: Enhance MongoDB storage with event publishing and outbox support
- Added `MongoAdvisoryObservationEventPublisher` and `NatsAdvisoryObservationEventPublisher` for event publishing. - Registered `IAdvisoryObservationEventPublisher` to choose between NATS and MongoDB based on configuration. - Introduced `MongoAdvisoryObservationEventOutbox` for outbox pattern implementation. - Updated service collection to include new event publishers and outbox. - Added a new hosted service `AdvisoryObservationTransportWorker` for processing events. feat: Update project dependencies - Added `NATS.Client.Core` package to the project for NATS integration. test: Add unit tests for AdvisoryLinkset normalization - Created `AdvisoryLinksetNormalizationConfidenceTests` to validate confidence score calculations. fix: Adjust confidence assertion in `AdvisoryObservationAggregationTests` - Updated confidence assertion to allow a range instead of a fixed value. test: Implement tests for AdvisoryObservationEventFactory - Added `AdvisoryObservationEventFactoryTests` to ensure correct mapping and hashing of observation events. chore: Configure test project for Findings Ledger - Created `Directory.Build.props` for test project configuration. - Added `StellaOps.Findings.Ledger.Exports.Unit.csproj` for unit tests related to findings ledger exports. feat: Implement export contracts for findings ledger - Defined export request and response contracts in `ExportContracts.cs`. - Created various export item records for findings, VEX, advisories, and SBOMs. feat: Add export functionality to Findings Ledger Web Service - Implemented endpoints for exporting findings, VEX, advisories, and SBOMs. - Integrated `ExportQueryService` for handling export logic and pagination. test: Add tests for Node language analyzer phase 22 - Implemented `NodePhase22SampleLoaderTests` to validate loading of NDJSON fixtures. - Created sample NDJSON file for testing. chore: Set up isolated test environment for Node tests - Added `node-isolated.runsettings` for isolated test execution. - Created `node-tests-isolated.sh` script for running tests in isolation.
This commit is contained in:
master
@@ -1,6 +1,8 @@
|
||||
using Microsoft.AspNetCore.Diagnostics;
|
||||
using Microsoft.AspNetCore.Http.HttpResults;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Serilog;
|
||||
using Serilog.Events;
|
||||
@@ -14,14 +16,17 @@ using StellaOps.Findings.Ledger.Infrastructure.Merkle;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Postgres;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Projection;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Policy;
|
||||
using StellaOps.Findings.Ledger.Infrastructure.Exports;
|
||||
using StellaOps.Findings.Ledger.Options;
|
||||
using StellaOps.Findings.Ledger.Services;
|
||||
using StellaOps.Findings.Ledger.WebService.Contracts;
|
||||
using StellaOps.Findings.Ledger.WebService.Mappings;
|
||||
using StellaOps.Findings.Ledger.WebService.Services;
|
||||
using StellaOps.Telemetry.Core;
|
||||
using StellaOps.Findings.Ledger.Services.Security;
|
||||
|
||||
const string LedgerWritePolicy = "ledger.events.write";
|
||||
const string LedgerExportPolicy = "ledger.export.read";
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
@@ -112,6 +117,13 @@ builder.Services.AddAuthorization(options =>
|
||||
policy.Requirements.Add(new StellaOpsScopeRequirement(scopes));
|
||||
policy.AddAuthenticationSchemes(StellaOpsAuthenticationDefaults.AuthenticationScheme);
|
||||
});
|
||||
|
||||
options.AddPolicy(LedgerExportPolicy, policy =>
|
||||
{
|
||||
policy.RequireAuthenticatedUser();
|
||||
policy.Requirements.Add(new StellaOpsScopeRequirement(scopes));
|
||||
policy.AddAuthenticationSchemes(StellaOpsAuthenticationDefaults.AuthenticationScheme);
|
||||
});
|
||||
});
|
||||
|
||||
builder.Services.AddSingleton<LedgerAnchorQueue>();
|
||||
@@ -133,6 +145,7 @@ builder.Services.AddSingleton<IAttachmentUrlSigner, AttachmentUrlSigner>();
|
||||
builder.Services.AddSingleton<IConsoleCsrfValidator, ConsoleCsrfValidator>();
|
||||
builder.Services.AddHostedService<LedgerMerkleAnchorWorker>();
|
||||
builder.Services.AddHostedService<LedgerProjectionWorker>();
|
||||
builder.Services.AddSingleton<ExportQueryService>();
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
@@ -197,6 +210,118 @@ app.MapPost("/vuln/ledger/events", async Task<Results<Created<LedgerEventRespons
|
||||
.ProducesProblem(StatusCodes.Status409Conflict)
|
||||
.ProducesProblem(StatusCodes.Status500InternalServerError);
|
||||
|
||||
app.MapGet("/ledger/export/findings", async Task<Results<FileStreamHttpResult, JsonHttpResult<ExportPage<FindingExportItem>>, ProblemHttpResult>> (
|
||||
HttpContext httpContext,
|
||||
ExportQueryService exportQueryService,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!httpContext.Request.Headers.TryGetValue("X-Stella-Tenant", out var tenantValues) || string.IsNullOrWhiteSpace(tenantValues))
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status400BadRequest, title: "missing_tenant", detail: "X-Stella-Tenant header is required.");
|
||||
}
|
||||
|
||||
var tenantId = tenantValues.ToString();
|
||||
var shape = httpContext.Request.Query["shape"].ToString();
|
||||
if (string.IsNullOrWhiteSpace(shape))
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status400BadRequest, title: "missing_shape", detail: "shape is required (canonical|compact).");
|
||||
}
|
||||
|
||||
var pageSize = exportQueryService.ClampPageSize(ParseInt(httpContext.Request.Query["page_size"]));
|
||||
|
||||
long? sinceSequence = ParseLong(httpContext.Request.Query["since_sequence"]);
|
||||
long? untilSequence = ParseLong(httpContext.Request.Query["until_sequence"]);
|
||||
DateTimeOffset? sinceObservedAt = ParseDate(httpContext.Request.Query["since_observed_at"]);
|
||||
DateTimeOffset? untilObservedAt = ParseDate(httpContext.Request.Query["until_observed_at"]);
|
||||
var status = httpContext.Request.Query["finding_status"].ToString();
|
||||
var severity = ParseDecimal(httpContext.Request.Query["severity"]);
|
||||
|
||||
var request = new ExportFindingsRequest(
|
||||
TenantId: tenantId,
|
||||
Shape: shape,
|
||||
SinceSequence: sinceSequence,
|
||||
UntilSequence: untilSequence,
|
||||
SinceObservedAt: sinceObservedAt,
|
||||
UntilObservedAt: untilObservedAt,
|
||||
Status: string.IsNullOrWhiteSpace(status) ? null : status,
|
||||
Severity: severity,
|
||||
PageSize: pageSize,
|
||||
FiltersHash: string.Empty,
|
||||
PagingKey: null);
|
||||
|
||||
var filtersHash = exportQueryService.ComputeFiltersHash(request);
|
||||
|
||||
ExportPagingKey? pagingKey = null;
|
||||
var pageToken = httpContext.Request.Query["page_token"].ToString();
|
||||
if (!string.IsNullOrWhiteSpace(pageToken))
|
||||
{
|
||||
if (!ExportPaging.TryParsePageToken(pageToken, filtersHash, out var parsedKey, out var error))
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status400BadRequest, title: error ?? "invalid_page_token");
|
||||
}
|
||||
|
||||
pagingKey = new ExportPagingKey(parsedKey!.SequenceNumber, parsedKey.PolicyVersion, parsedKey.CycleHash);
|
||||
}
|
||||
|
||||
request = request with { FiltersHash = filtersHash, PagingKey = pagingKey };
|
||||
|
||||
ExportPage<FindingExportItem> page;
|
||||
try
|
||||
{
|
||||
page = await exportQueryService.GetFindingsAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (InvalidOperationException ex) when (ex.Message == "filters_hash_mismatch")
|
||||
{
|
||||
return TypedResults.Problem(statusCode: StatusCodes.Status400BadRequest, title: "page_token_filters_mismatch");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(page.NextPageToken))
|
||||
{
|
||||
httpContext.Response.Headers["X-Stella-Next-Page-Token"] = page.NextPageToken;
|
||||
}
|
||||
httpContext.Response.Headers["X-Stella-Result-Count"] = page.Items.Count.ToString();
|
||||
|
||||
var acceptsNdjson = httpContext.Request.Headers.Accept.Any(h => h.Contains("application/x-ndjson", StringComparison.OrdinalIgnoreCase));
|
||||
if (acceptsNdjson)
|
||||
{
|
||||
httpContext.Response.ContentType = "application/x-ndjson";
|
||||
var stream = new MemoryStream();
|
||||
await using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { SkipValidation = false, Indented = false });
|
||||
foreach (var item in page.Items)
|
||||
{
|
||||
JsonSerializer.Serialize(writer, item);
|
||||
writer.Flush();
|
||||
await stream.WriteAsync(new byte[] { (byte)'\n' }, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
stream.Position = 0;
|
||||
return TypedResults.Stream(stream, contentType: "application/x-ndjson");
|
||||
}
|
||||
|
||||
return TypedResults.Json(page);
|
||||
})
|
||||
.WithName("LedgerExportFindings")
|
||||
.RequireAuthorization(LedgerExportPolicy)
|
||||
.Produces(StatusCodes.Status200OK)
|
||||
.ProducesProblem(StatusCodes.Status400BadRequest)
|
||||
.ProducesProblem(StatusCodes.Status401Unauthorized)
|
||||
.ProducesProblem(StatusCodes.Status403Forbidden)
|
||||
.ProducesProblem(StatusCodes.Status500InternalServerError);
|
||||
|
||||
app.MapGet("/ledger/export/vex", () => TypedResults.Json(new ExportPage<VexExportItem>(Array.Empty<VexExportItem>(), null)))
|
||||
.WithName("LedgerExportVex")
|
||||
.RequireAuthorization(LedgerExportPolicy)
|
||||
.Produces(StatusCodes.Status200OK);
|
||||
|
||||
app.MapGet("/ledger/export/advisories", () => TypedResults.Json(new ExportPage<AdvisoryExportItem>(Array.Empty<AdvisoryExportItem>(), null)))
|
||||
.WithName("LedgerExportAdvisories")
|
||||
.RequireAuthorization(LedgerExportPolicy)
|
||||
.Produces(StatusCodes.Status200OK);
|
||||
|
||||
app.MapGet("/ledger/export/sboms", () => TypedResults.Json(new ExportPage<SbomExportItem>(Array.Empty<SbomExportItem>(), null)))
|
||||
.WithName("LedgerExportSboms")
|
||||
.RequireAuthorization(LedgerExportPolicy)
|
||||
.Produces(StatusCodes.Status200OK);
|
||||
|
||||
app.Run();
|
||||
|
||||
static Created<LedgerEventResponse> CreateCreatedResponse(LedgerEventRecord record)
|
||||
|
||||
Reference in New Issue
Block a user