feat: Implement air-gap functionality with timeline impact and evidence snapshot services
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Mirror Thin Bundle Sign & Verify / mirror-sign (push) Has been cancelled

- Added AirgapTimelineImpact, AirgapTimelineImpactInput, and AirgapTimelineImpactResult records for managing air-gap bundle import impacts.
- Introduced EvidenceSnapshotRecord, EvidenceSnapshotLinkInput, and EvidenceSnapshotLinkResult records for linking findings to evidence snapshots.
- Created IEvidenceSnapshotRepository interface for managing evidence snapshot records.
- Developed StalenessValidationService to validate staleness and enforce freshness thresholds.
- Implemented AirgapTimelineService for emitting timeline events related to bundle imports.
- Added EvidenceSnapshotService for linking findings to evidence snapshots and verifying their validity.
- Introduced AirGapOptions for configuring air-gap staleness enforcement and thresholds.
- Added minimal jsPDF stub for offline/testing builds in the web application.
- Created TypeScript definitions for jsPDF to enhance type safety in the web application.
This commit is contained in:
StellaOps Bot
2025-12-06 01:30:08 +02:00
parent 6c1177a6ce
commit 2eaf0f699b
144 changed files with 7578 additions and 2581 deletions

View File

@@ -1,3 +1,4 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using StellaOps.Concelier.Models.Observations;
using StellaOps.Concelier.RawModels;
@@ -20,3 +21,13 @@ public sealed record AdvisoryObservationLinksetAggregateResponse(
ImmutableArray<RawRelationship> Relationships,
double Confidence,
ImmutableArray<AdvisoryLinksetConflict> Conflicts);
/// <summary>
/// Request to publish observation events to NATS/Redis.
/// </summary>
public sealed record ObservationEventPublishRequest(IReadOnlyList<string>? ObservationIds);
/// <summary>
/// Request to publish linkset events to NATS/Redis.
/// </summary>
public sealed record LinksetEventPublishRequest(IReadOnlyList<string>? AdvisoryIds);

View File

@@ -0,0 +1,133 @@
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.WebService.Contracts;
/// <summary>
/// Hybrid RFC 7807 + Standard Error Envelope.
/// Per CONCELIER-WEB-OAS-61-002.
/// </summary>
/// <remarks>
/// Combines RFC 7807 Problem Details format with a structured error code
/// for machine-readable error handling. This enables both human-readable
/// problem descriptions and programmatic error code checking.
/// </remarks>
public sealed record ErrorEnvelope
{
/// <summary>
/// A URI reference that identifies the problem type (RFC 7807).
/// </summary>
[JsonPropertyName("type")]
public required string Type { get; init; }
/// <summary>
/// A short, human-readable summary of the problem type (RFC 7807).
/// </summary>
[JsonPropertyName("title")]
public required string Title { get; init; }
/// <summary>
/// The HTTP status code (RFC 7807).
/// </summary>
[JsonPropertyName("status")]
public required int Status { get; init; }
/// <summary>
/// A human-readable explanation specific to this occurrence (RFC 7807).
/// </summary>
[JsonPropertyName("detail")]
public string? Detail { get; init; }
/// <summary>
/// A URI reference that identifies the specific occurrence (RFC 7807).
/// </summary>
[JsonPropertyName("instance")]
public string? Instance { get; init; }
/// <summary>
/// Distributed trace identifier for correlation.
/// </summary>
[JsonPropertyName("traceId")]
public string? TraceId { get; init; }
/// <summary>
/// Structured error details with machine-readable code.
/// </summary>
[JsonPropertyName("error")]
public ErrorDetail? Error { get; init; }
}
/// <summary>
/// Structured error detail with machine-readable code.
/// </summary>
public sealed record ErrorDetail
{
/// <summary>
/// Machine-readable error code (e.g., "VALIDATION_FAILED", "RESOURCE_NOT_FOUND").
/// </summary>
[JsonPropertyName("code")]
public required string Code { get; init; }
/// <summary>
/// Human-readable error message.
/// </summary>
[JsonPropertyName("message")]
public string? Message { get; init; }
/// <summary>
/// Target of the error (field name, resource identifier, etc.).
/// </summary>
[JsonPropertyName("target")]
public string? Target { get; init; }
/// <summary>
/// Additional metadata about the error.
/// </summary>
[JsonPropertyName("metadata")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IReadOnlyDictionary<string, object?>? Metadata { get; init; }
/// <summary>
/// Nested validation errors for complex validation failures.
/// </summary>
[JsonPropertyName("innerErrors")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IReadOnlyList<ValidationError>? InnerErrors { get; init; }
/// <summary>
/// URL for more information about this error.
/// </summary>
[JsonPropertyName("helpUrl")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public string? HelpUrl { get; init; }
/// <summary>
/// Retry-after hint in seconds (for rate limiting).
/// </summary>
[JsonPropertyName("retryAfter")]
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? RetryAfter { get; init; }
}
/// <summary>
/// Individual validation error for field-level issues.
/// </summary>
public sealed record ValidationError
{
/// <summary>
/// Field path (e.g., "advisoryId", "data.severity").
/// </summary>
[JsonPropertyName("field")]
public required string Field { get; init; }
/// <summary>
/// Error code for this specific validation error.
/// </summary>
[JsonPropertyName("code")]
public required string Code { get; init; }
/// <summary>
/// Human-readable message for this validation error.
/// </summary>
[JsonPropertyName("message")]
public string? Message { get; init; }
}

View File

@@ -0,0 +1,148 @@
namespace StellaOps.Concelier.WebService.Diagnostics;
/// <summary>
/// Machine-readable error codes for API responses.
/// Per CONCELIER-WEB-OAS-61-002.
/// </summary>
public static class ErrorCodes
{
// ─────────────────────────────────────────────────────────────────────────
// Validation Errors (4xx)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>Generic validation failure.</summary>
public const string ValidationFailed = "VALIDATION_FAILED";
/// <summary>Required field is missing.</summary>
public const string RequiredFieldMissing = "REQUIRED_FIELD_MISSING";
/// <summary>Field value is invalid.</summary>
public const string InvalidFieldValue = "INVALID_FIELD_VALUE";
/// <summary>Tenant ID is required but not provided.</summary>
public const string TenantRequired = "TENANT_REQUIRED";
/// <summary>Advisory ID is required but not provided.</summary>
public const string AdvisoryIdRequired = "ADVISORY_ID_REQUIRED";
/// <summary>Vulnerability key is required but not provided.</summary>
public const string VulnerabilityKeyRequired = "VULNERABILITY_KEY_REQUIRED";
/// <summary>Cursor parameter must be an integer.</summary>
public const string InvalidCursor = "INVALID_CURSOR";
/// <summary>Invalid pagination parameters.</summary>
public const string InvalidPagination = "INVALID_PAGINATION";
// ─────────────────────────────────────────────────────────────────────────
// Resource Errors (404)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>Requested resource was not found.</summary>
public const string ResourceNotFound = "RESOURCE_NOT_FOUND";
/// <summary>Advisory not found.</summary>
public const string AdvisoryNotFound = "ADVISORY_NOT_FOUND";
/// <summary>Vulnerability not found.</summary>
public const string VulnerabilityNotFound = "VULNERABILITY_NOT_FOUND";
/// <summary>Evidence not found.</summary>
public const string EvidenceNotFound = "EVIDENCE_NOT_FOUND";
/// <summary>Tenant not found.</summary>
public const string TenantNotFound = "TENANT_NOT_FOUND";
/// <summary>Job not found.</summary>
public const string JobNotFound = "JOB_NOT_FOUND";
/// <summary>Mirror not found.</summary>
public const string MirrorNotFound = "MIRROR_NOT_FOUND";
/// <summary>Bundle source not found.</summary>
public const string BundleSourceNotFound = "BUNDLE_SOURCE_NOT_FOUND";
// ─────────────────────────────────────────────────────────────────────────
// AOC (Aggregation-Only Contract) Errors
// ─────────────────────────────────────────────────────────────────────────
/// <summary>AOC violation occurred.</summary>
public const string AocViolation = "AOC_VIOLATION";
/// <summary>Forbidden field in advisory (ERR_AOC_001).</summary>
public const string AocForbiddenField = "AOC_FORBIDDEN_FIELD";
/// <summary>Merge attempt detected (ERR_AOC_002).</summary>
public const string AocMergeAttempt = "AOC_MERGE_ATTEMPT";
/// <summary>Derived field modification (ERR_AOC_006).</summary>
public const string AocDerivedField = "AOC_DERIVED_FIELD";
/// <summary>Unknown field detected (ERR_AOC_007).</summary>
public const string AocUnknownField = "AOC_UNKNOWN_FIELD";
// ─────────────────────────────────────────────────────────────────────────
// Conflict Errors (409)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>Resource already exists.</summary>
public const string ResourceConflict = "RESOURCE_CONFLICT";
/// <summary>Concurrent modification detected.</summary>
public const string ConcurrencyConflict = "CONCURRENCY_CONFLICT";
/// <summary>Lease already held by another client.</summary>
public const string LeaseConflict = "LEASE_CONFLICT";
// ─────────────────────────────────────────────────────────────────────────
// State Errors (423 Locked)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>Resource is locked.</summary>
public const string ResourceLocked = "RESOURCE_LOCKED";
/// <summary>Lease rejected.</summary>
public const string LeaseRejected = "LEASE_REJECTED";
// ─────────────────────────────────────────────────────────────────────────
// AirGap/Sealed Mode Errors
// ─────────────────────────────────────────────────────────────────────────
/// <summary>AirGap mode is disabled.</summary>
public const string AirGapDisabled = "AIRGAP_DISABLED";
/// <summary>Sealed mode violation.</summary>
public const string SealedModeViolation = "SEALED_MODE_VIOLATION";
/// <summary>Source blocked by sealed mode.</summary>
public const string SourceBlocked = "SOURCE_BLOCKED";
// ─────────────────────────────────────────────────────────────────────────
// Rate Limiting (429)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>Rate limit exceeded.</summary>
public const string RateLimitExceeded = "RATE_LIMIT_EXCEEDED";
/// <summary>Quota exceeded.</summary>
public const string QuotaExceeded = "QUOTA_EXCEEDED";
// ─────────────────────────────────────────────────────────────────────────
// Server Errors (5xx)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>Internal server error.</summary>
public const string InternalError = "INTERNAL_ERROR";
/// <summary>Service unavailable.</summary>
public const string ServiceUnavailable = "SERVICE_UNAVAILABLE";
/// <summary>Job execution failure.</summary>
public const string JobFailure = "JOB_FAILURE";
/// <summary>External service failure.</summary>
public const string ExternalServiceFailure = "EXTERNAL_SERVICE_FAILURE";
/// <summary>Database operation failed.</summary>
public const string DatabaseError = "DATABASE_ERROR";
}

View File

@@ -0,0 +1,165 @@
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.Core.AirGap;
using StellaOps.Concelier.Core.AirGap.Models;
using StellaOps.Concelier.WebService.Diagnostics;
using StellaOps.Concelier.WebService.Options;
using StellaOps.Concelier.WebService.Results;
namespace StellaOps.Concelier.WebService.Extensions;
/// <summary>
/// Endpoint extensions for AirGap functionality.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
internal static class AirGapEndpointExtensions
{
public static void MapConcelierAirGapEndpoints(this WebApplication app)
{
var group = app.MapGroup("/api/v1/concelier/airgap")
.WithTags("AirGap");
// GET /api/v1/concelier/airgap/catalog - Aggregated bundle catalog
group.MapGet("/catalog", async (
HttpContext context,
IBundleCatalogService catalogService,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
[FromQuery] string? cursor,
[FromQuery] int? limit,
CancellationToken cancellationToken) =>
{
var airGapOptions = optionsMonitor.CurrentValue.AirGap;
if (!airGapOptions.Enabled)
{
return ConcelierProblemResultFactory.AirGapDisabled(context);
}
var catalog = await catalogService.GetCatalogAsync(cursor, limit, cancellationToken)
.ConfigureAwait(false);
return Results.Ok(catalog);
});
// GET /api/v1/concelier/airgap/sources - List registered sources
group.MapGet("/sources", (
HttpContext context,
IBundleSourceRegistry sourceRegistry,
IOptionsMonitor<ConcelierOptions> optionsMonitor) =>
{
var airGapOptions = optionsMonitor.CurrentValue.AirGap;
if (!airGapOptions.Enabled)
{
return ConcelierProblemResultFactory.AirGapDisabled(context);
}
var sources = sourceRegistry.GetSources();
return Results.Ok(new { sources, count = sources.Count });
});
// POST /api/v1/concelier/airgap/sources - Register new source
group.MapPost("/sources", async (
HttpContext context,
IBundleSourceRegistry sourceRegistry,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
[FromBody] BundleSourceRegistration registration,
CancellationToken cancellationToken) =>
{
var airGapOptions = optionsMonitor.CurrentValue.AirGap;
if (!airGapOptions.Enabled)
{
return ConcelierProblemResultFactory.AirGapDisabled(context);
}
if (string.IsNullOrWhiteSpace(registration.Id))
{
return ConcelierProblemResultFactory.RequiredFieldMissing(context, "id");
}
var source = await sourceRegistry.RegisterAsync(registration, cancellationToken)
.ConfigureAwait(false);
return Results.Created($"/api/v1/concelier/airgap/sources/{source.Id}", source);
});
// GET /api/v1/concelier/airgap/sources/{sourceId} - Get specific source
group.MapGet("/sources/{sourceId}", (
HttpContext context,
IBundleSourceRegistry sourceRegistry,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
string sourceId) =>
{
var airGapOptions = optionsMonitor.CurrentValue.AirGap;
if (!airGapOptions.Enabled)
{
return ConcelierProblemResultFactory.AirGapDisabled(context);
}
var source = sourceRegistry.GetSource(sourceId);
if (source is null)
{
return ConcelierProblemResultFactory.BundleSourceNotFound(context, sourceId);
}
return Results.Ok(source);
});
// DELETE /api/v1/concelier/airgap/sources/{sourceId} - Unregister source
group.MapDelete("/sources/{sourceId}", async (
HttpContext context,
IBundleSourceRegistry sourceRegistry,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
string sourceId,
CancellationToken cancellationToken) =>
{
var airGapOptions = optionsMonitor.CurrentValue.AirGap;
if (!airGapOptions.Enabled)
{
return ConcelierProblemResultFactory.AirGapDisabled(context);
}
var removed = await sourceRegistry.UnregisterAsync(sourceId, cancellationToken)
.ConfigureAwait(false);
return removed
? Results.NoContent()
: ConcelierProblemResultFactory.BundleSourceNotFound(context, sourceId);
});
// POST /api/v1/concelier/airgap/sources/{sourceId}/validate - Validate source
group.MapPost("/sources/{sourceId}/validate", async (
HttpContext context,
IBundleSourceRegistry sourceRegistry,
IOptionsMonitor<ConcelierOptions> optionsMonitor,
string sourceId,
CancellationToken cancellationToken) =>
{
var airGapOptions = optionsMonitor.CurrentValue.AirGap;
if (!airGapOptions.Enabled)
{
return ConcelierProblemResultFactory.AirGapDisabled(context);
}
var result = await sourceRegistry.ValidateAsync(sourceId, cancellationToken)
.ConfigureAwait(false);
return Results.Ok(result);
});
// GET /api/v1/concelier/airgap/status - Sealed-mode status
group.MapGet("/status", (
HttpContext context,
ISealedModeEnforcer sealedModeEnforcer,
IOptionsMonitor<ConcelierOptions> optionsMonitor) =>
{
var airGapOptions = optionsMonitor.CurrentValue.AirGap;
if (!airGapOptions.Enabled)
{
return ConcelierProblemResultFactory.AirGapDisabled(context);
}
var status = sealedModeEnforcer.GetStatus();
return Results.Ok(status);
});
}
}

View File

@@ -1,9 +1,11 @@
using System.Globalization;
using System.IO;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.WebService.Options;
using StellaOps.Concelier.WebService.Services;
using System.Globalization;
using System.IO;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Options;
using StellaOps.Concelier.WebService.Diagnostics;
using StellaOps.Concelier.WebService.Options;
using StellaOps.Concelier.WebService.Results;
using StellaOps.Concelier.WebService.Services;
namespace StellaOps.Concelier.WebService.Extensions;
@@ -24,7 +26,7 @@ internal static class MirrorEndpointExtensions
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
if (!mirrorOptions.Enabled)
{
return Results.NotFound();
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (!TryAuthorize(mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult))
@@ -35,15 +37,15 @@ internal static class MirrorEndpointExtensions
if (!limiter.TryAcquire("__index__", IndexScope, mirrorOptions.MaxIndexRequestsPerHour, out var retryAfter))
{
ApplyRetryAfter(context.Response, retryAfter);
return Results.StatusCode(StatusCodes.Status429TooManyRequests);
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
}
if (!locator.TryResolveIndex(out var path, out _))
{
return Results.NotFound();
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
return await WriteFileAsync(path, context.Response, "application/json").ConfigureAwait(false);
return await WriteFileAsync(context, path, "application/json").ConfigureAwait(false);
});
app.MapGet("/concelier/exports/{**relativePath}", async (
@@ -57,17 +59,17 @@ internal static class MirrorEndpointExtensions
var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions();
if (!mirrorOptions.Enabled)
{
return Results.NotFound();
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (string.IsNullOrWhiteSpace(relativePath))
{
return Results.NotFound();
return ConcelierProblemResultFactory.MirrorNotFound(context);
}
if (!locator.TryResolveRelativePath(relativePath, out var path, out _, out var domainId))
{
return Results.NotFound();
return ConcelierProblemResultFactory.MirrorNotFound(context, relativePath);
}
var domain = FindDomain(mirrorOptions, domainId);
@@ -81,11 +83,11 @@ internal static class MirrorEndpointExtensions
if (!limiter.TryAcquire(domain?.Id ?? "__mirror__", DownloadScope, limit, out var retryAfter))
{
ApplyRetryAfter(context.Response, retryAfter);
return Results.StatusCode(StatusCodes.Status429TooManyRequests);
return ConcelierProblemResultFactory.RateLimitExceeded(context, (int?)retryAfter?.TotalSeconds);
}
var contentType = ResolveContentType(path);
return await WriteFileAsync(path, context.Response, contentType).ConfigureAwait(false);
return await WriteFileAsync(context, path, contentType).ConfigureAwait(false);
});
}
@@ -112,12 +114,12 @@ internal static class MirrorEndpointExtensions
return null;
}
private static bool TryAuthorize(bool requireAuthentication, bool enforceAuthority, HttpContext context, bool authorityConfigured, out IResult result)
{
result = Results.Empty;
if (!requireAuthentication)
{
return true;
private static bool TryAuthorize(bool requireAuthentication, bool enforceAuthority, HttpContext context, bool authorityConfigured, out IResult result)
{
result = Results.Empty;
if (!requireAuthentication)
{
return true;
}
if (!enforceAuthority || !authorityConfigured)
@@ -128,19 +130,19 @@ internal static class MirrorEndpointExtensions
if (context.User?.Identity?.IsAuthenticated == true)
{
return true;
}
context.Response.Headers.WWWAuthenticate = "Bearer realm=\"StellaOps Concelier Mirror\"";
result = Results.StatusCode(StatusCodes.Status401Unauthorized);
return false;
}
private static Task<IResult> WriteFileAsync(string path, HttpResponse response, string contentType)
{
}
context.Response.Headers.WWWAuthenticate = "Bearer realm=\"StellaOps Concelier Mirror\"";
result = Results.StatusCode(StatusCodes.Status401Unauthorized);
return false;
}
private static Task<IResult> WriteFileAsync(HttpContext context, string path, string contentType)
{
var fileInfo = new FileInfo(path);
if (!fileInfo.Exists)
{
return Task.FromResult(Results.NotFound());
return Task.FromResult(ConcelierProblemResultFactory.MirrorNotFound(context, path));
}
var stream = new FileStream(
@@ -149,12 +151,12 @@ internal static class MirrorEndpointExtensions
FileAccess.Read,
FileShare.Read | FileShare.Delete);
response.Headers.CacheControl = BuildCacheControlHeader(path);
response.Headers.LastModified = fileInfo.LastWriteTimeUtc.ToString("R", CultureInfo.InvariantCulture);
response.ContentLength = fileInfo.Length;
return Task.FromResult(Results.Stream(stream, contentType));
}
context.Response.Headers.CacheControl = BuildCacheControlHeader(path);
context.Response.Headers.LastModified = fileInfo.LastWriteTimeUtc.ToString("R", CultureInfo.InvariantCulture);
context.Response.ContentLength = fileInfo.Length;
return Task.FromResult(Results.Stream(stream, contentType));
}
private static string ResolveContentType(string path)
{
if (path.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
@@ -178,28 +180,28 @@ internal static class MirrorEndpointExtensions
}
var seconds = Math.Max((int)Math.Ceiling(retryAfter.Value.TotalSeconds), 1);
response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture);
}
private static string BuildCacheControlHeader(string path)
{
var fileName = Path.GetFileName(path);
if (fileName is null)
{
return "public, max-age=60";
}
if (string.Equals(fileName, "index.json", StringComparison.OrdinalIgnoreCase))
{
return "public, max-age=60";
}
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ||
fileName.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
{
return "public, max-age=300, immutable";
}
return "public, max-age=300";
}
}
response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture);
}
private static string BuildCacheControlHeader(string path)
{
var fileName = Path.GetFileName(path);
if (fileName is null)
{
return "public, max-age=60";
}
if (string.Equals(fileName, "index.json", StringComparison.OrdinalIgnoreCase))
{
return "public, max-age=60";
}
if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ||
fileName.EndsWith(".jws", StringComparison.OrdinalIgnoreCase))
{
return "public, max-age=300, immutable";
}
return "public, max-age=300";
}
}

View File

@@ -0,0 +1,158 @@
using System.Text.Json.Serialization;
namespace StellaOps.Concelier.WebService.Options;
/// <summary>
/// Air-gap configuration options for Concelier.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed class AirGapOptions
{
/// <summary>
/// Enable air-gap mode with bundle-based feed consumption.
/// </summary>
public bool Enabled { get; set; }
/// <summary>
/// Sealed mode configuration (blocks direct internet feeds when enabled).
/// </summary>
public SealedModeOptions SealedMode { get; set; } = new();
/// <summary>
/// Bundle sources configuration.
/// </summary>
public BundleSourcesOptions Sources { get; set; } = new();
/// <summary>
/// Catalog configuration.
/// </summary>
public CatalogOptions Catalog { get; set; } = new();
/// <summary>
/// Sealed mode configuration options.
/// When sealed mode is enabled, direct internet feeds are blocked.
/// </summary>
public sealed class SealedModeOptions
{
/// <summary>
/// Enable sealed mode (block direct internet feeds).
/// </summary>
public bool Enabled { get; set; }
/// <summary>
/// List of sources explicitly allowed even in sealed mode.
/// </summary>
public IList<string> AllowedSources { get; set; } = new List<string>();
/// <summary>
/// List of hosts that are allowed for egress even in sealed mode.
/// Useful for internal mirrors or private registries.
/// </summary>
public IList<string> AllowedHosts { get; set; } = new List<string>();
/// <summary>
/// Warn-only mode: log violations but don't block requests.
/// Useful for testing sealed mode before full enforcement.
/// </summary>
public bool WarnOnly { get; set; }
}
/// <summary>
/// Bundle sources configuration options.
/// </summary>
public sealed class BundleSourcesOptions
{
/// <summary>
/// Root directory for bundle storage.
/// </summary>
public string Root { get; set; } = "bundles";
/// <summary>
/// Automatically register sources from bundle directory on startup.
/// </summary>
public bool AutoDiscovery { get; set; } = true;
/// <summary>
/// File patterns to match for auto-discovery.
/// </summary>
public IList<string> DiscoveryPatterns { get; set; } = new List<string> { "*.bundle.json", "catalog.json" };
/// <summary>
/// Pre-configured bundle sources.
/// </summary>
public IList<BundleSourceConfig> Configured { get; set; } = new List<BundleSourceConfig>();
/// <summary>
/// Computed absolute path to root directory.
/// </summary>
[JsonIgnore]
public string RootAbsolute { get; internal set; } = string.Empty;
}
/// <summary>
/// Configuration for a single bundle source.
/// </summary>
public sealed class BundleSourceConfig
{
/// <summary>
/// Unique identifier for the source.
/// </summary>
public string Id { get; set; } = string.Empty;
/// <summary>
/// Display name for the source.
/// </summary>
public string? DisplayName { get; set; }
/// <summary>
/// Source type (directory, archive, remote).
/// </summary>
public string Type { get; set; } = "directory";
/// <summary>
/// Path or URL to the bundle source.
/// </summary>
public string Location { get; set; } = string.Empty;
/// <summary>
/// Enable this source.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Priority for this source (lower = higher priority).
/// </summary>
public int Priority { get; set; } = 100;
/// <summary>
/// Verification mode for bundles from this source.
/// </summary>
public string VerificationMode { get; set; } = "signature";
}
/// <summary>
/// Catalog configuration options.
/// </summary>
public sealed class CatalogOptions
{
/// <summary>
/// Enable catalog aggregation from all sources.
/// </summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// Cache duration for aggregated catalog in seconds.
/// </summary>
public int CacheDurationSeconds { get; set; } = 300;
/// <summary>
/// Maximum number of items per catalog page.
/// </summary>
public int MaxPageSize { get; set; } = 100;
/// <summary>
/// Include bundle provenance in catalog responses.
/// </summary>
public bool IncludeProvenance { get; set; } = true;
}
}

View File

@@ -27,6 +27,12 @@ public sealed class ConcelierOptions
public StellaOpsCryptoOptions Crypto { get; } = new();
/// <summary>
/// Air-gap mode configuration.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public AirGapOptions AirGap { get; set; } = new();
public sealed class StorageOptions
{
public string Driver { get; set; } = "mongo";

View File

@@ -51,6 +51,7 @@ using StellaOps.Aoc;
using StellaOps.Aoc.AspNetCore.Routing;
using StellaOps.Aoc.AspNetCore.Results;
using StellaOps.Concelier.WebService.Contracts;
using StellaOps.Concelier.WebService.Results;
using StellaOps.Concelier.Core.Aoc;
using StellaOps.Concelier.Core.Raw;
using StellaOps.Concelier.RawModels;
@@ -712,7 +713,7 @@ var observationsEndpoint = app.MapGet("/concelier/observations", async (
{"reason", "format"},
{"stage", "ingest"}
});
return Results.BadRequest(ex.Message);
return ConcelierProblemResultFactory.ValidationFailed(context, ex.Message);
}
var elapsed = stopwatch.Elapsed;
@@ -867,7 +868,7 @@ app.MapGet("/v1/lnm/linksets/{advisoryId}", async (
if (string.IsNullOrWhiteSpace(advisoryId))
{
return Results.BadRequest("advisoryId is required.");
return ConcelierProblemResultFactory.AdvisoryIdRequired(context);
}
var stopwatch = Stopwatch.StartNew();
@@ -880,7 +881,7 @@ app.MapGet("/v1/lnm/linksets/{advisoryId}", async (
if (result.Linksets.IsDefaultOrEmpty)
{
return Results.NotFound();
return ConcelierProblemResultFactory.AdvisoryNotFound(context, advisoryId);
}
var linkset = result.Linksets[0];
@@ -1178,7 +1179,7 @@ var advisoryRawGetEndpoint = app.MapGet("/advisories/raw/{id}", async (
var record = await rawService.FindByIdAsync(tenant, id.Trim(), cancellationToken).ConfigureAwait(false);
if (record is null)
{
return Results.NotFound();
return ConcelierProblemResultFactory.AdvisoryNotFound(context, id);
}
var response = new AdvisoryRawRecordResponse(
@@ -1222,7 +1223,7 @@ var advisoryRawProvenanceEndpoint = app.MapGet("/advisories/raw/{id}/provenance"
var record = await rawService.FindByIdAsync(tenant, id.Trim(), cancellationToken).ConfigureAwait(false);
if (record is null)
{
return Results.NotFound();
return ConcelierProblemResultFactory.AdvisoryNotFound(context, id);
}
var response = new AdvisoryRawProvenanceResponse(
@@ -1241,6 +1242,379 @@ if (authorityConfigured)
advisoryRawProvenanceEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
// Advisory observations endpoint - filtered by alias/purl/source with strict tenant scopes.
// Echoes upstream values + provenance fields only (no merge-derived judgments).
var advisoryObservationsEndpoint = app.MapGet("/advisories/observations", async (
HttpContext context,
[FromServices] IAdvisoryObservationQueryService observationService,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
var authorizationError = EnsureTenantAuthorized(context, tenant);
if (authorizationError is not null)
{
return authorizationError;
}
var query = context.Request.Query;
// Parse query parameters
var aliases = query.TryGetValue("alias", out var aliasValues)
? AdvisoryRawRequestMapper.NormalizeStrings(aliasValues)
: null;
var purls = query.TryGetValue("purl", out var purlValues)
? AdvisoryRawRequestMapper.NormalizeStrings(purlValues)
: null;
var cpes = query.TryGetValue("cpe", out var cpeValues)
? AdvisoryRawRequestMapper.NormalizeStrings(cpeValues)
: null;
var observationIds = query.TryGetValue("id", out var idValues)
? AdvisoryRawRequestMapper.NormalizeStrings(idValues)
: null;
int? limit = null;
if (query.TryGetValue("limit", out var limitValues) &&
int.TryParse(limitValues.FirstOrDefault(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedLimit) &&
parsedLimit > 0)
{
limit = Math.Min(parsedLimit, 200); // Cap at 200
}
string? cursor = null;
if (query.TryGetValue("cursor", out var cursorValues))
{
var cursorValue = cursorValues.FirstOrDefault();
if (!string.IsNullOrWhiteSpace(cursorValue))
{
cursor = cursorValue.Trim();
}
}
// Build query options with tenant scope
var options = new AdvisoryObservationQueryOptions(
tenant,
observationIds: observationIds,
aliases: aliases,
purls: purls,
cpes: cpes,
limit: limit,
cursor: cursor);
var result = await observationService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
// Map to response contracts
var linksetResponse = new AdvisoryObservationLinksetAggregateResponse(
result.Linkset.Aliases,
result.Linkset.Purls,
result.Linkset.Cpes,
result.Linkset.References,
result.Linkset.Scopes,
result.Linkset.Relationships,
result.Linkset.Confidence,
result.Linkset.Conflicts);
var response = new AdvisoryObservationQueryResponse(
result.Observations,
linksetResponse,
result.NextCursor,
result.HasMore);
return JsonResult(response);
}).WithName("GetAdvisoryObservations");
if (authorityConfigured)
{
advisoryObservationsEndpoint.RequireAuthorization(ObservationsPolicyName);
}
// Advisory linksets endpoint - surfaces correlation + conflict payloads with ERR_AGG_* mapping.
// No synthesis/merge - echoes upstream values only.
var advisoryLinksetsEndpoint = app.MapGet("/advisories/linksets", async (
HttpContext context,
[FromServices] IAdvisoryLinksetQueryService linksetService,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
var authorizationError = EnsureTenantAuthorized(context, tenant);
if (authorizationError is not null)
{
return authorizationError;
}
var query = context.Request.Query;
// Parse advisory IDs (alias values like CVE-*, GHSA-*)
var advisoryIds = query.TryGetValue("advisoryId", out var advisoryIdValues)
? AdvisoryRawRequestMapper.NormalizeStrings(advisoryIdValues)
: (query.TryGetValue("alias", out var aliasValues)
? AdvisoryRawRequestMapper.NormalizeStrings(aliasValues)
: null);
var sources = query.TryGetValue("source", out var sourceValues)
? AdvisoryRawRequestMapper.NormalizeStrings(sourceValues)
: null;
int? limit = null;
if (query.TryGetValue("limit", out var limitValues) &&
int.TryParse(limitValues.FirstOrDefault(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedLimit) &&
parsedLimit > 0)
{
limit = Math.Min(parsedLimit, 500); // Cap at 500
}
string? cursor = null;
if (query.TryGetValue("cursor", out var cursorValues))
{
var cursorValue = cursorValues.FirstOrDefault();
if (!string.IsNullOrWhiteSpace(cursorValue))
{
cursor = cursorValue.Trim();
}
}
var options = new AdvisoryLinksetQueryOptions(
tenant,
advisoryIds,
sources,
limit,
cursor);
var result = await linksetService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
// Map to LNM linkset response format
var items = result.Linksets.Select(linkset => new LnmLinksetResponse(
linkset.AdvisoryId,
linkset.Source,
linkset.Normalized?.Purls ?? Array.Empty<string>(),
linkset.Normalized?.Cpes ?? Array.Empty<string>(),
null, // Summary not available in linkset
null, // PublishedAt
null, // ModifiedAt
null, // Severity - no derived judgment
null, // Status
linkset.Provenance is not null
? new LnmLinksetProvenance(
linkset.CreatedAt,
null, // ConnectorId
linkset.Provenance.ObservationHashes?.FirstOrDefault(),
null) // DsseEnvelopeHash
: null,
linkset.Conflicts?.Select(c => new LnmLinksetConflict(
c.Field,
c.Reason,
c.Values?.FirstOrDefault(),
null,
null)).ToArray() ?? Array.Empty<LnmLinksetConflict>(),
Array.Empty<LnmLinksetTimeline>(),
linkset.Normalized is not null
? new LnmLinksetNormalized(
null, // Aliases not in normalized
linkset.Normalized.Purls,
linkset.Normalized.Cpes,
linkset.Normalized.Versions,
null) // Ranges serialized differently
: null,
false, // Not from cache
Array.Empty<string>(),
linkset.ObservationIds.ToArray())).ToArray();
var response = new LnmLinksetPage(items, 1, items.Length, null);
return JsonResult(response);
}).WithName("GetAdvisoryLinksets");
if (authorityConfigured)
{
advisoryLinksetsEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
// Advisory linksets export endpoint for evidence bundles
var advisoryLinksetsExportEndpoint = app.MapGet("/advisories/linksets/export", async (
HttpContext context,
[FromServices] IAdvisoryLinksetQueryService linksetService,
[FromServices] TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
var authorizationError = EnsureTenantAuthorized(context, tenant);
if (authorizationError is not null)
{
return authorizationError;
}
var query = context.Request.Query;
var advisoryIds = query.TryGetValue("advisoryId", out var advisoryIdValues)
? AdvisoryRawRequestMapper.NormalizeStrings(advisoryIdValues)
: null;
var sources = query.TryGetValue("source", out var sourceValues)
? AdvisoryRawRequestMapper.NormalizeStrings(sourceValues)
: null;
var options = new AdvisoryLinksetQueryOptions(tenant, advisoryIds, sources, 1000, null);
var result = await linksetService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
// Export format with provenance metadata
var exportItems = result.Linksets.Select(linkset => new
{
advisoryId = linkset.AdvisoryId,
source = linkset.Source,
tenantId = linkset.TenantId,
observationIds = linkset.ObservationIds.ToArray(),
confidence = linkset.Confidence,
conflicts = linkset.Conflicts?.Select(c => new
{
field = c.Field,
reason = c.Reason,
values = c.Values,
sourceIds = c.SourceIds
}).ToArray(),
normalized = linkset.Normalized is not null ? new
{
purls = linkset.Normalized.Purls,
cpes = linkset.Normalized.Cpes,
versions = linkset.Normalized.Versions
} : null,
provenance = linkset.Provenance is not null ? new
{
observationHashes = linkset.Provenance.ObservationHashes,
toolVersion = linkset.Provenance.ToolVersion,
policyHash = linkset.Provenance.PolicyHash
} : null,
createdAt = linkset.CreatedAt,
builtByJobId = linkset.BuiltByJobId
}).ToArray();
var export = new
{
tenant = tenant,
exportedAt = timeProvider.GetUtcNow(),
count = exportItems.Length,
hasMore = result.HasMore,
linksets = exportItems
};
return JsonResult(export);
}).WithName("ExportAdvisoryLinksets");
if (authorityConfigured)
{
advisoryLinksetsExportEndpoint.RequireAuthorization(AdvisoryReadPolicyName);
}
// Internal endpoint for publishing observation events to NATS/Redis.
// Publishes advisory.observation.updated@1 events with tenant + provenance references only.
app.MapPost("/internal/events/observations/publish", async (
HttpContext context,
[FromBody] ObservationEventPublishRequest request,
[FromServices] IAdvisoryObservationQueryService observationService,
[FromServices] IAdvisoryObservationEventPublisher? eventPublisher,
[FromServices] TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
if (eventPublisher is null)
{
return Problem(context, "Event publishing not configured", StatusCodes.Status503ServiceUnavailable, ProblemTypes.ServiceUnavailable, "Event publisher service is not available.");
}
if (request?.ObservationIds is null || request.ObservationIds.Count == 0)
{
return Problem(context, "observationIds required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide at least one observation ID.");
}
var options = new AdvisoryObservationQueryOptions(tenant, observationIds: request.ObservationIds);
var result = await observationService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
var published = 0;
foreach (var observation in result.Observations)
{
var @event = AdvisoryObservationUpdatedEvent.FromObservation(
observation,
supersedesId: null,
traceId: context.TraceIdentifier);
await eventPublisher.PublishAsync(@event, cancellationToken).ConfigureAwait(false);
published++;
}
return Results.Ok(new { tenant, published, requestedCount = request.ObservationIds.Count, timestamp = timeProvider.GetUtcNow() });
}).WithName("PublishObservationEvents");
// Internal endpoint for publishing linkset events to NATS/Redis.
// Publishes advisory.linkset.updated@1 events with idempotent keys and tenant + provenance references.
app.MapPost("/internal/events/linksets/publish", async (
HttpContext context,
[FromBody] LinksetEventPublishRequest request,
[FromServices] IAdvisoryLinksetQueryService linksetService,
[FromServices] IAdvisoryLinksetEventPublisher? eventPublisher,
[FromServices] TimeProvider timeProvider,
CancellationToken cancellationToken) =>
{
ApplyNoCache(context.Response);
if (!TryResolveTenant(context, requireHeader: true, out var tenant, out var tenantError))
{
return tenantError;
}
if (eventPublisher is null)
{
return Problem(context, "Event publishing not configured", StatusCodes.Status503ServiceUnavailable, ProblemTypes.ServiceUnavailable, "Event publisher service is not available.");
}
if (request?.AdvisoryIds is null || request.AdvisoryIds.Count == 0)
{
return Problem(context, "advisoryIds required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide at least one advisory ID.");
}
var options = new AdvisoryLinksetQueryOptions(tenant, request.AdvisoryIds, null, 500);
var result = await linksetService.QueryAsync(options, cancellationToken).ConfigureAwait(false);
var published = 0;
foreach (var linkset in result.Linksets)
{
var linksetId = $"{linkset.TenantId}:{linkset.Source}:{linkset.AdvisoryId}";
var @event = AdvisoryLinksetUpdatedEvent.FromLinkset(
linkset,
previousLinkset: null,
linksetId: linksetId,
traceId: context.TraceIdentifier);
await eventPublisher.PublishAsync(@event, cancellationToken).ConfigureAwait(false);
published++;
}
return Results.Ok(new { tenant, published, requestedCount = request.AdvisoryIds.Count, hasMore = result.HasMore, timestamp = timeProvider.GetUtcNow() });
}).WithName("PublishLinksetEvents");
var advisoryEvidenceEndpoint = app.MapGet("/vuln/evidence/advisories/{advisoryKey}", async (
string advisoryKey,
HttpContext context,
@@ -1743,7 +2117,7 @@ var advisorySummaryEndpoint = app.MapGet("/advisories/summary", async (
}
catch (FormatException ex)
{
return Results.BadRequest(ex.Message);
return ConcelierProblemResultFactory.ValidationFailed(context, ex.Message);
}
var items = queryResult.Linksets
@@ -1947,13 +2321,13 @@ app.MapGet("/concelier/advisories/{vulnerabilityKey}/replay", async (
{
if (string.IsNullOrWhiteSpace(vulnerabilityKey))
{
return Results.BadRequest("vulnerabilityKey must be provided.");
return ConcelierProblemResultFactory.VulnerabilityKeyRequired(context);
}
var replay = await eventLog.ReplayAsync(vulnerabilityKey.Trim(), asOf, cancellationToken).ConfigureAwait(false);
if (replay.Statements.Length == 0 && replay.Conflicts.Length == 0)
{
return Results.NotFound();
return ConcelierProblemResultFactory.VulnerabilityNotFound(context, vulnerabilityKey);
}
var response = new
@@ -2309,7 +2683,7 @@ IResult JsonResult<T>(T value, int? statusCode = null)
return Results.Content(payload, "application/json", Encoding.UTF8, statusCode);
}
IResult Problem(HttpContext context, string title, int statusCode, string type, string? detail = null, IDictionary<string, object?>? extensions = null)
IResult Problem(HttpContext context, string title, int statusCode, string type, string? detail = null, IDictionary<string, object?>? extensions = null, string? errorCode = null)
{
var traceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier;
extensions ??= new Dictionary<string, object?>(StringComparer.Ordinal)
@@ -2322,6 +2696,12 @@ IResult Problem(HttpContext context, string title, int statusCode, string type,
extensions["traceId"] = traceId;
}
// Per CONCELIER-WEB-OAS-61-002: Add error code extension for machine-readable errors
if (!string.IsNullOrEmpty(errorCode))
{
extensions["error"] = new { code = errorCode, message = detail ?? title };
}
var problemDetails = new ProblemDetails
{
Type = type,
@@ -3208,7 +3588,7 @@ var concelierTimelineEndpoint = app.MapGet("/obs/concelier/timeline", async (
var candidateCursor = cursor ?? context.Request.Headers["Last-Event-ID"].FirstOrDefault();
if (!string.IsNullOrWhiteSpace(candidateCursor) && !int.TryParse(candidateCursor, NumberStyles.Integer, CultureInfo.InvariantCulture, out startId))
{
return Results.BadRequest(new { error = "cursor must be integer" });
return ConcelierProblemResultFactory.InvalidCursor(context);
}
var logger = loggerFactory.CreateLogger("ConcelierTimeline");

View File

@@ -0,0 +1,398 @@
using System.Diagnostics;
using Microsoft.AspNetCore.Http;
using StellaOps.Concelier.WebService.Contracts;
using StellaOps.Concelier.WebService.Diagnostics;
namespace StellaOps.Concelier.WebService.Results;
/// <summary>
/// Factory for creating standardized error responses.
/// Per CONCELIER-WEB-OAS-61-002.
/// </summary>
public static class ConcelierProblemResultFactory
{
/// <summary>
/// Creates a standardized Problem response with error code.
/// </summary>
public static IResult Problem(
HttpContext context,
string type,
string title,
int statusCode,
string errorCode,
string? detail = null,
string? target = null,
IReadOnlyDictionary<string, object?>? metadata = null,
IReadOnlyList<ValidationError>? innerErrors = null)
{
var envelope = new ErrorEnvelope
{
Type = type,
Title = title,
Status = statusCode,
Detail = detail,
Instance = context.Request.Path,
TraceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier,
Error = new ErrorDetail
{
Code = errorCode,
Message = detail ?? title,
Target = target,
Metadata = metadata,
InnerErrors = innerErrors
}
};
return Microsoft.AspNetCore.Http.Results.Json(envelope, statusCode: statusCode);
}
// ─────────────────────────────────────────────────────────────────────────
// Validation Errors (400)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>
/// Creates a 400 Bad Request response for validation failure.
/// </summary>
public static IResult ValidationFailed(
HttpContext context,
string detail,
string? target = null,
IReadOnlyList<ValidationError>? innerErrors = null)
{
return Problem(
context,
ProblemTypes.Validation,
"Validation failed",
StatusCodes.Status400BadRequest,
ErrorCodes.ValidationFailed,
detail,
target,
innerErrors: innerErrors);
}
/// <summary>
/// Creates a 400 Bad Request response for required field missing.
/// </summary>
public static IResult RequiredFieldMissing(
HttpContext context,
string fieldName)
{
return Problem(
context,
ProblemTypes.Validation,
"Required field missing",
StatusCodes.Status400BadRequest,
ErrorCodes.RequiredFieldMissing,
$"{fieldName} is required.",
fieldName);
}
/// <summary>
/// Creates a 400 Bad Request response for advisory ID required.
/// </summary>
public static IResult AdvisoryIdRequired(HttpContext context)
{
return Problem(
context,
ProblemTypes.Validation,
"Advisory ID required",
StatusCodes.Status400BadRequest,
ErrorCodes.AdvisoryIdRequired,
"advisoryId is required.",
"advisoryId");
}
/// <summary>
/// Creates a 400 Bad Request response for vulnerability key required.
/// </summary>
public static IResult VulnerabilityKeyRequired(HttpContext context)
{
return Problem(
context,
ProblemTypes.Validation,
"Vulnerability key required",
StatusCodes.Status400BadRequest,
ErrorCodes.VulnerabilityKeyRequired,
"vulnerabilityKey must be provided.",
"vulnerabilityKey");
}
/// <summary>
/// Creates a 400 Bad Request response for invalid cursor.
/// </summary>
public static IResult InvalidCursor(HttpContext context)
{
return Problem(
context,
ProblemTypes.Validation,
"Invalid cursor",
StatusCodes.Status400BadRequest,
ErrorCodes.InvalidCursor,
"cursor must be an integer.",
"cursor");
}
// ─────────────────────────────────────────────────────────────────────────
// Not Found Errors (404)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>
/// Creates a 404 Not Found response for resource not found.
/// </summary>
public static IResult NotFound(
HttpContext context,
string errorCode,
string resourceType,
string? resourceId = null)
{
var detail = resourceId is not null
? $"{resourceType} '{resourceId}' not found."
: $"{resourceType} not found.";
return Problem(
context,
ProblemTypes.NotFound,
$"{resourceType} not found",
StatusCodes.Status404NotFound,
errorCode,
detail,
resourceId);
}
/// <summary>
/// Creates a 404 Not Found response for advisory not found.
/// </summary>
public static IResult AdvisoryNotFound(HttpContext context, string? advisoryId = null)
{
return NotFound(context, ErrorCodes.AdvisoryNotFound, "Advisory", advisoryId);
}
/// <summary>
/// Creates a 404 Not Found response for vulnerability not found.
/// </summary>
public static IResult VulnerabilityNotFound(HttpContext context, string? vulnerabilityKey = null)
{
return NotFound(context, ErrorCodes.VulnerabilityNotFound, "Vulnerability", vulnerabilityKey);
}
/// <summary>
/// Creates a 404 Not Found response for evidence not found.
/// </summary>
public static IResult EvidenceNotFound(HttpContext context, string? evidenceId = null)
{
return NotFound(context, ErrorCodes.EvidenceNotFound, "Evidence", evidenceId);
}
/// <summary>
/// Creates a 404 Not Found response for mirror not found.
/// </summary>
public static IResult MirrorNotFound(HttpContext context, string? mirrorId = null)
{
return NotFound(context, ErrorCodes.MirrorNotFound, "Mirror", mirrorId);
}
/// <summary>
/// Creates a 404 Not Found response for bundle source not found.
/// </summary>
public static IResult BundleSourceNotFound(HttpContext context, string? sourceId = null)
{
return NotFound(context, ErrorCodes.BundleSourceNotFound, "Bundle source", sourceId);
}
/// <summary>
/// Creates a generic 404 Not Found response.
/// </summary>
public static IResult ResourceNotFound(HttpContext context, string? detail = null)
{
return Problem(
context,
ProblemTypes.NotFound,
"Resource not found",
StatusCodes.Status404NotFound,
ErrorCodes.ResourceNotFound,
detail ?? "The requested resource was not found.");
}
// ─────────────────────────────────────────────────────────────────────────
// Conflict Errors (409)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>
/// Creates a 409 Conflict response.
/// </summary>
public static IResult Conflict(
HttpContext context,
string errorCode,
string detail,
string? target = null)
{
return Problem(
context,
ProblemTypes.Conflict,
"Conflict",
StatusCodes.Status409Conflict,
errorCode,
detail,
target);
}
/// <summary>
/// Creates a 409 Conflict response for lease conflict.
/// </summary>
public static IResult LeaseConflict(HttpContext context, string detail)
{
return Conflict(context, ErrorCodes.LeaseConflict, detail);
}
// ─────────────────────────────────────────────────────────────────────────
// Locked Errors (423)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>
/// Creates a 423 Locked response.
/// </summary>
public static IResult Locked(
HttpContext context,
string errorCode,
string detail)
{
return Problem(
context,
ProblemTypes.Locked,
"Resource locked",
StatusCodes.Status423Locked,
errorCode,
detail);
}
/// <summary>
/// Creates a 423 Locked response for lease rejection.
/// </summary>
public static IResult LeaseRejected(HttpContext context, string detail)
{
return Problem(
context,
ProblemTypes.LeaseRejected,
"Lease rejected",
StatusCodes.Status423Locked,
ErrorCodes.LeaseRejected,
detail);
}
// ─────────────────────────────────────────────────────────────────────────
// AirGap/Sealed Mode Errors
// ─────────────────────────────────────────────────────────────────────────
/// <summary>
/// Creates a 404 Not Found response for AirGap disabled.
/// </summary>
public static IResult AirGapDisabled(HttpContext context)
{
return Problem(
context,
"https://stellaops.org/problems/airgap-disabled",
"AirGap mode disabled",
StatusCodes.Status404NotFound,
ErrorCodes.AirGapDisabled,
"AirGap mode is not enabled on this instance.");
}
/// <summary>
/// Creates a 403 Forbidden response for sealed mode violation.
/// </summary>
public static IResult SealedModeViolation(
HttpContext context,
string sourceName,
string destination)
{
return Problem(
context,
"https://stellaops.org/problems/sealed-violation",
"Sealed mode violation",
StatusCodes.Status403Forbidden,
ErrorCodes.SealedModeViolation,
$"Source '{sourceName}' is not allowed to access '{destination}' in sealed mode.",
sourceName,
new Dictionary<string, object?> { ["destination"] = destination });
}
// ─────────────────────────────────────────────────────────────────────────
// Rate Limiting (429)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>
/// Creates a 429 Too Many Requests response.
/// </summary>
public static IResult RateLimitExceeded(HttpContext context, int? retryAfterSeconds = null)
{
var envelope = new ErrorEnvelope
{
Type = "https://stellaops.org/problems/rate-limit",
Title = "Rate limit exceeded",
Status = StatusCodes.Status429TooManyRequests,
Detail = "Too many requests. Please try again later.",
Instance = context.Request.Path,
TraceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier,
Error = new ErrorDetail
{
Code = ErrorCodes.RateLimitExceeded,
Message = "Too many requests. Please try again later.",
RetryAfter = retryAfterSeconds
}
};
return Microsoft.AspNetCore.Http.Results.Json(envelope, statusCode: StatusCodes.Status429TooManyRequests);
}
// ─────────────────────────────────────────────────────────────────────────
// Server Errors (5xx)
// ─────────────────────────────────────────────────────────────────────────
/// <summary>
/// Creates a 500 Internal Server Error response.
/// </summary>
public static IResult InternalError(
HttpContext context,
string? detail = null)
{
return Problem(
context,
"https://stellaops.org/problems/internal-error",
"Internal server error",
StatusCodes.Status500InternalServerError,
ErrorCodes.InternalError,
detail ?? "An unexpected error occurred.");
}
/// <summary>
/// Creates a 503 Service Unavailable response.
/// </summary>
public static IResult ServiceUnavailable(
HttpContext context,
string? detail = null)
{
return Problem(
context,
ProblemTypes.ServiceUnavailable,
"Service unavailable",
StatusCodes.Status503ServiceUnavailable,
ErrorCodes.ServiceUnavailable,
detail ?? "The service is temporarily unavailable.");
}
/// <summary>
/// Creates a 500 response for job failure.
/// </summary>
public static IResult JobFailure(
HttpContext context,
string detail)
{
return Problem(
context,
ProblemTypes.JobFailure,
"Job failure",
StatusCodes.Status500InternalServerError,
ErrorCodes.JobFailure,
detail);
}
}

View File

@@ -0,0 +1,77 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
namespace StellaOps.Concelier.Core.AirGap;
/// <summary>
/// Service collection extensions for AirGap services.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public static class AirGapServiceCollectionExtensions
{
/// <summary>
/// Adds AirGap services to the service collection.
/// </summary>
/// <param name="services">The service collection.</param>
/// <param name="configureSealed">Optional sealed mode configuration.</param>
/// <returns>The service collection for chaining.</returns>
public static IServiceCollection AddConcelierAirGapServices(
this IServiceCollection services,
Action<SealedModeConfiguration>? configureSealed = null)
{
ArgumentNullException.ThrowIfNull(services);
// Register TimeProvider if not already registered
services.TryAddSingleton(TimeProvider.System);
// Register core services
services.TryAddSingleton<IBundleSourceRegistry, BundleSourceRegistry>();
services.TryAddSingleton<IBundleCatalogService, BundleCatalogService>();
// Configure and register sealed mode enforcer
var sealedConfig = new SealedModeConfiguration();
configureSealed?.Invoke(sealedConfig);
services.TryAddSingleton<ISealedModeEnforcer>(sp =>
{
var logger = sp.GetRequiredService<Microsoft.Extensions.Logging.ILogger<SealedModeEnforcer>>();
var timeProvider = sp.GetService<TimeProvider>();
return new SealedModeEnforcer(
logger,
isSealed: sealedConfig.IsSealed,
warnOnly: sealedConfig.WarnOnly,
allowedSources: sealedConfig.AllowedSources,
allowedHosts: sealedConfig.AllowedHosts,
timeProvider: timeProvider);
});
return services;
}
}
/// <summary>
/// Configuration for sealed mode.
/// </summary>
public sealed class SealedModeConfiguration
{
/// <summary>
/// Enable sealed mode.
/// </summary>
public bool IsSealed { get; set; }
/// <summary>
/// Enable warn-only mode (log violations but don't block).
/// </summary>
public bool WarnOnly { get; set; }
/// <summary>
/// Sources allowed even in sealed mode.
/// </summary>
public IList<string> AllowedSources { get; } = new List<string>();
/// <summary>
/// Hosts allowed even in sealed mode.
/// </summary>
public IList<string> AllowedHosts { get; } = new List<string>();
}

View File

@@ -0,0 +1,250 @@
using System.Collections.Immutable;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.AirGap.Models;
namespace StellaOps.Concelier.Core.AirGap;
/// <summary>
/// Default implementation of <see cref="IBundleCatalogService"/>.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed class BundleCatalogService : IBundleCatalogService
{
private readonly IBundleSourceRegistry _sourceRegistry;
private readonly ILogger<BundleCatalogService> _logger;
private readonly TimeProvider _timeProvider;
private readonly int _defaultPageSize;
private readonly int _maxPageSize;
private AggregatedCatalog? _cachedCatalog;
private DateTimeOffset _cacheExpiry = DateTimeOffset.MinValue;
private readonly object _cacheLock = new();
public BundleCatalogService(
IBundleSourceRegistry sourceRegistry,
ILogger<BundleCatalogService> logger,
TimeProvider? timeProvider = null,
int defaultPageSize = 50,
int maxPageSize = 100)
{
_sourceRegistry = sourceRegistry ?? throw new ArgumentNullException(nameof(sourceRegistry));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_defaultPageSize = defaultPageSize;
_maxPageSize = maxPageSize;
}
/// <inheritdoc />
public async Task<AggregatedCatalog> GetCatalogAsync(
string? cursor = null,
int? limit = null,
CancellationToken cancellationToken = default)
{
var fullCatalog = await GetOrRefreshCatalogAsync(cancellationToken).ConfigureAwait(false);
return ApplyPagination(fullCatalog, cursor, limit);
}
/// <inheritdoc />
public async Task<AggregatedCatalog> GetCatalogBySourceAsync(
string sourceId,
string? cursor = null,
int? limit = null,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
var fullCatalog = await GetOrRefreshCatalogAsync(cancellationToken).ConfigureAwait(false);
var filteredEntries = fullCatalog.Entries
.Where(e => string.Equals(e.SourceId, sourceId, StringComparison.OrdinalIgnoreCase))
.ToImmutableArray();
var filteredCatalog = fullCatalog with
{
Entries = filteredEntries,
TotalCount = filteredEntries.Length,
SourceIds = ImmutableArray.Create(sourceId)
};
return ApplyPagination(filteredCatalog, cursor, limit);
}
/// <inheritdoc />
public async Task<BundleCatalogEntry?> GetBundleAsync(
string bundleId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(bundleId);
var catalog = await GetOrRefreshCatalogAsync(cancellationToken).ConfigureAwait(false);
return catalog.Entries.FirstOrDefault(e =>
string.Equals(e.BundleId, bundleId, StringComparison.OrdinalIgnoreCase));
}
/// <inheritdoc />
public Task RefreshAsync(CancellationToken cancellationToken = default)
{
lock (_cacheLock)
{
_cachedCatalog = null;
_cacheExpiry = DateTimeOffset.MinValue;
}
_logger.LogDebug("Catalog cache invalidated");
return Task.CompletedTask;
}
private async Task<AggregatedCatalog> GetOrRefreshCatalogAsync(CancellationToken cancellationToken)
{
var now = _timeProvider.GetUtcNow();
lock (_cacheLock)
{
if (_cachedCatalog is not null && now < _cacheExpiry)
{
return _cachedCatalog;
}
}
var catalog = await BuildCatalogAsync(cancellationToken).ConfigureAwait(false);
lock (_cacheLock)
{
_cachedCatalog = catalog;
_cacheExpiry = now.AddMinutes(5); // Default 5-minute cache
}
return catalog;
}
private Task<AggregatedCatalog> BuildCatalogAsync(CancellationToken cancellationToken)
{
var sources = _sourceRegistry.GetSources()
.Where(s => s.Enabled && s.Status != BundleSourceStatus.Error)
.ToList();
var entries = new List<BundleCatalogEntry>();
var sourceIds = new List<string>();
foreach (var source in sources)
{
var sourceEntries = DiscoverBundles(source);
entries.AddRange(sourceEntries);
sourceIds.Add(source.Id);
}
var now = _timeProvider.GetUtcNow();
var etag = ComputeETag(entries);
_logger.LogDebug(
"Built catalog with {EntryCount} entries from {SourceCount} sources",
entries.Count, sources.Count);
return Task.FromResult(new AggregatedCatalog
{
Entries = entries.OrderBy(e => e.BundleId).ToImmutableArray(),
TotalCount = entries.Count,
SourceIds = sourceIds.ToImmutableArray(),
ComputedAt = now,
ETag = etag
});
}
private IEnumerable<BundleCatalogEntry> DiscoverBundles(BundleSourceInfo source)
{
// Actual implementation would discover bundles from the source
// For now, return empty - this would be expanded based on source type
return source.Type switch
{
"directory" => DiscoverDirectoryBundles(source),
"archive" => DiscoverArchiveBundles(source),
"remote" => Enumerable.Empty<BundleCatalogEntry>(), // Would require async HTTP calls
_ => Enumerable.Empty<BundleCatalogEntry>()
};
}
private IEnumerable<BundleCatalogEntry> DiscoverDirectoryBundles(BundleSourceInfo source)
{
if (!Directory.Exists(source.Location))
{
yield break;
}
foreach (var file in Directory.EnumerateFiles(source.Location, "*.bundle.json", SearchOption.AllDirectories))
{
var fileInfo = new FileInfo(file);
var bundleId = Path.GetFileNameWithoutExtension(fileInfo.Name);
yield return new BundleCatalogEntry
{
BundleId = bundleId,
SourceId = source.Id,
Type = "advisory", // Would be parsed from bundle metadata
ContentHash = $"sha256:{ComputeFileHash(file)}",
SizeBytes = fileInfo.Length,
CreatedAt = fileInfo.CreationTimeUtc,
ModifiedAt = fileInfo.LastWriteTimeUtc
};
}
}
private IEnumerable<BundleCatalogEntry> DiscoverArchiveBundles(BundleSourceInfo source)
{
// Would extract and inspect archive contents
yield break;
}
private AggregatedCatalog ApplyPagination(AggregatedCatalog catalog, string? cursor, int? limit)
{
var pageSize = Math.Min(limit ?? _defaultPageSize, _maxPageSize);
var offset = ParseCursor(cursor);
var pagedEntries = catalog.Entries
.Skip(offset)
.Take(pageSize)
.ToImmutableArray();
string? nextCursor = null;
if (offset + pageSize < catalog.TotalCount)
{
nextCursor = (offset + pageSize).ToString();
}
return catalog with
{
Entries = pagedEntries,
NextCursor = nextCursor
};
}
private static int ParseCursor(string? cursor)
{
if (string.IsNullOrEmpty(cursor))
{
return 0;
}
return int.TryParse(cursor, out var offset) ? offset : 0;
}
private static string ComputeETag(IEnumerable<BundleCatalogEntry> entries)
{
var builder = new StringBuilder();
foreach (var entry in entries.OrderBy(e => e.BundleId))
{
builder.Append(entry.BundleId);
builder.Append(entry.ContentHash);
}
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString()));
return $"W/\"{Convert.ToHexString(hash)[..16]}\"";
}
private static string ComputeFileHash(string filePath)
{
using var stream = File.OpenRead(filePath);
var hash = SHA256.HashData(stream);
return Convert.ToHexString(hash).ToLowerInvariant();
}
}

View File

@@ -0,0 +1,185 @@
using System.Collections.Concurrent;
using System.Collections.Immutable;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.AirGap.Models;
namespace StellaOps.Concelier.Core.AirGap;
/// <summary>
/// Default implementation of <see cref="IBundleSourceRegistry"/>.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed class BundleSourceRegistry : IBundleSourceRegistry
{
private readonly ConcurrentDictionary<string, BundleSourceInfo> _sources = new(StringComparer.OrdinalIgnoreCase);
private readonly ILogger<BundleSourceRegistry> _logger;
private readonly TimeProvider _timeProvider;
public BundleSourceRegistry(
ILogger<BundleSourceRegistry> logger,
TimeProvider? timeProvider = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <inheritdoc />
public IReadOnlyList<BundleSourceInfo> GetSources()
=> _sources.Values.OrderBy(s => s.Priority).ThenBy(s => s.Id).ToList();
/// <inheritdoc />
public BundleSourceInfo? GetSource(string sourceId)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
return _sources.GetValueOrDefault(sourceId);
}
/// <inheritdoc />
public Task<BundleSourceInfo> RegisterAsync(
BundleSourceRegistration registration,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(registration);
ArgumentException.ThrowIfNullOrWhiteSpace(registration.Id);
ArgumentException.ThrowIfNullOrWhiteSpace(registration.Type);
ArgumentException.ThrowIfNullOrWhiteSpace(registration.Location);
var now = _timeProvider.GetUtcNow();
var sourceInfo = new BundleSourceInfo
{
Id = registration.Id,
DisplayName = registration.DisplayName,
Type = registration.Type,
Location = registration.Location,
Enabled = registration.Enabled,
Priority = registration.Priority,
VerificationMode = registration.VerificationMode,
RegisteredAt = now,
Status = BundleSourceStatus.Unknown,
Metadata = ImmutableDictionary<string, string>.Empty
};
_sources[registration.Id] = sourceInfo;
_logger.LogInformation(
"Registered bundle source: {SourceId}, type={Type}, location={Location}",
registration.Id, registration.Type, registration.Location);
return Task.FromResult(sourceInfo);
}
/// <inheritdoc />
public Task<bool> UnregisterAsync(string sourceId, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
var removed = _sources.TryRemove(sourceId, out _);
if (removed)
{
_logger.LogInformation("Unregistered bundle source: {SourceId}", sourceId);
}
return Task.FromResult(removed);
}
/// <inheritdoc />
public Task<BundleSourceValidationResult> ValidateAsync(
string sourceId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
if (!_sources.TryGetValue(sourceId, out var source))
{
return Task.FromResult(BundleSourceValidationResult.Failure(sourceId, $"Source '{sourceId}' not found"));
}
var now = _timeProvider.GetUtcNow();
// Basic validation - actual implementation would check source accessibility
var result = source.Type switch
{
"directory" => ValidateDirectorySource(source),
"archive" => ValidateArchiveSource(source),
"remote" => ValidateRemoteSource(source),
_ => BundleSourceValidationResult.Failure(sourceId, $"Unknown source type: {source.Type}")
};
// Update source status
var updatedSource = source with
{
LastValidatedAt = now,
Status = result.Status,
BundleCount = result.BundleCount,
ErrorMessage = result.Errors.Length > 0 ? string.Join("; ", result.Errors) : null
};
_sources[sourceId] = updatedSource;
_logger.LogDebug(
"Validated bundle source: {SourceId}, status={Status}, bundles={BundleCount}",
sourceId, result.Status, result.BundleCount);
return Task.FromResult(result);
}
/// <inheritdoc />
public Task<bool> SetEnabledAsync(string sourceId, bool enabled, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
if (!_sources.TryGetValue(sourceId, out var source))
{
return Task.FromResult(false);
}
var updatedSource = source with
{
Enabled = enabled,
Status = enabled ? source.Status : BundleSourceStatus.Disabled
};
_sources[sourceId] = updatedSource;
_logger.LogInformation("Set bundle source {SourceId} enabled={Enabled}", sourceId, enabled);
return Task.FromResult(true);
}
private BundleSourceValidationResult ValidateDirectorySource(BundleSourceInfo source)
{
if (!Directory.Exists(source.Location))
{
return BundleSourceValidationResult.Failure(source.Id, $"Directory not found: {source.Location}");
}
var bundleFiles = Directory.GetFiles(source.Location, "*.bundle.json", SearchOption.AllDirectories);
return BundleSourceValidationResult.Success(source.Id, bundleFiles.Length);
}
private BundleSourceValidationResult ValidateArchiveSource(BundleSourceInfo source)
{
if (!File.Exists(source.Location))
{
return BundleSourceValidationResult.Failure(source.Id, $"Archive not found: {source.Location}");
}
// Actual implementation would inspect archive contents
return BundleSourceValidationResult.Success(source.Id, 0);
}
private BundleSourceValidationResult ValidateRemoteSource(BundleSourceInfo source)
{
if (!Uri.TryCreate(source.Location, UriKind.Absolute, out var uri))
{
return BundleSourceValidationResult.Failure(source.Id, $"Invalid URL: {source.Location}");
}
// Actual implementation would check remote accessibility
return new BundleSourceValidationResult
{
SourceId = source.Id,
IsValid = true,
Status = BundleSourceStatus.Unknown,
ValidatedAt = _timeProvider.GetUtcNow(),
Warnings = ImmutableArray.Create("Remote validation not implemented - assuming valid")
};
}
}

View File

@@ -0,0 +1,39 @@
using StellaOps.Concelier.Core.AirGap.Models;
namespace StellaOps.Concelier.Core.AirGap;
/// <summary>
/// Service for accessing the aggregated bundle catalog.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public interface IBundleCatalogService
{
/// <summary>
/// Gets the aggregated catalog from all sources.
/// </summary>
Task<AggregatedCatalog> GetCatalogAsync(
string? cursor = null,
int? limit = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets catalog entries for a specific source.
/// </summary>
Task<AggregatedCatalog> GetCatalogBySourceAsync(
string sourceId,
string? cursor = null,
int? limit = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets a specific bundle entry.
/// </summary>
Task<BundleCatalogEntry?> GetBundleAsync(
string bundleId,
CancellationToken cancellationToken = default);
/// <summary>
/// Refreshes the catalog cache.
/// </summary>
Task RefreshAsync(CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,44 @@
using StellaOps.Concelier.Core.AirGap.Models;
namespace StellaOps.Concelier.Core.AirGap;
/// <summary>
/// Registry for managing bundle sources in air-gap mode.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public interface IBundleSourceRegistry
{
/// <summary>
/// Gets all registered sources.
/// </summary>
IReadOnlyList<BundleSourceInfo> GetSources();
/// <summary>
/// Gets a specific source by ID.
/// </summary>
BundleSourceInfo? GetSource(string sourceId);
/// <summary>
/// Registers a new bundle source.
/// </summary>
Task<BundleSourceInfo> RegisterAsync(
BundleSourceRegistration registration,
CancellationToken cancellationToken = default);
/// <summary>
/// Unregisters a bundle source.
/// </summary>
Task<bool> UnregisterAsync(string sourceId, CancellationToken cancellationToken = default);
/// <summary>
/// Validates a bundle source.
/// </summary>
Task<BundleSourceValidationResult> ValidateAsync(
string sourceId,
CancellationToken cancellationToken = default);
/// <summary>
/// Enables or disables a source.
/// </summary>
Task<bool> SetEnabledAsync(string sourceId, bool enabled, CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,52 @@
using StellaOps.Concelier.Core.AirGap.Models;
namespace StellaOps.Concelier.Core.AirGap;
/// <summary>
/// Enforces sealed mode by blocking direct internet feeds.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public interface ISealedModeEnforcer
{
/// <summary>
/// Gets whether sealed mode is currently active.
/// </summary>
bool IsSealed { get; }
/// <summary>
/// Ensures a source is allowed to access the given destination.
/// Throws <see cref="SealedModeViolationException"/> if not allowed and not in warn-only mode.
/// </summary>
void EnsureSourceAllowed(string sourceName, Uri destination);
/// <summary>
/// Checks if a source is allowed to access the given destination.
/// </summary>
bool IsSourceAllowed(string sourceName, Uri destination);
/// <summary>
/// Gets the list of currently blocked sources.
/// </summary>
IReadOnlyList<string> GetBlockedSources();
/// <summary>
/// Gets the current sealed mode status.
/// </summary>
SealedModeStatus GetStatus();
}
/// <summary>
/// Exception thrown when a sealed mode violation occurs.
/// </summary>
public sealed class SealedModeViolationException : Exception
{
public SealedModeViolationException(string sourceName, Uri destination)
: base($"Sealed mode violation: source '{sourceName}' attempted to access '{destination}'")
{
SourceName = sourceName;
Destination = destination;
}
public string SourceName { get; }
public Uri Destination { get; }
}

View File

@@ -0,0 +1,40 @@
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.AirGap.Models;
/// <summary>
/// Aggregated bundle catalog from all sources.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed record AggregatedCatalog
{
/// <summary>
/// Catalog entries.
/// </summary>
public ImmutableArray<BundleCatalogEntry> Entries { get; init; } = ImmutableArray<BundleCatalogEntry>.Empty;
/// <summary>
/// Total number of entries (may differ from Entries.Length if paginated).
/// </summary>
public int TotalCount { get; init; }
/// <summary>
/// Sources that contributed to this catalog.
/// </summary>
public ImmutableArray<string> SourceIds { get; init; } = ImmutableArray<string>.Empty;
/// <summary>
/// When the catalog was computed.
/// </summary>
public DateTimeOffset ComputedAt { get; init; }
/// <summary>
/// Catalog version/ETag for caching.
/// </summary>
public string? ETag { get; init; }
/// <summary>
/// Cursor for pagination.
/// </summary>
public string? NextCursor { get; init; }
}

View File

@@ -0,0 +1,117 @@
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.AirGap.Models;
/// <summary>
/// Entry in the aggregated bundle catalog.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed record BundleCatalogEntry
{
/// <summary>
/// Bundle identifier.
/// </summary>
public required string BundleId { get; init; }
/// <summary>
/// Source that provides this bundle.
/// </summary>
public required string SourceId { get; init; }
/// <summary>
/// Bundle type (advisory, vex, sbom, etc.).
/// </summary>
public required string Type { get; init; }
/// <summary>
/// Bundle version.
/// </summary>
public string? Version { get; init; }
/// <summary>
/// Content hash for integrity verification.
/// </summary>
public required string ContentHash { get; init; }
/// <summary>
/// Size of the bundle in bytes.
/// </summary>
public long SizeBytes { get; init; }
/// <summary>
/// When the bundle was created.
/// </summary>
public DateTimeOffset CreatedAt { get; init; }
/// <summary>
/// When the bundle was last modified.
/// </summary>
public DateTimeOffset? ModifiedAt { get; init; }
/// <summary>
/// Number of items in the bundle.
/// </summary>
public int ItemCount { get; init; }
/// <summary>
/// Bundle metadata.
/// </summary>
public ImmutableDictionary<string, string> Metadata { get; init; } = ImmutableDictionary<string, string>.Empty;
/// <summary>
/// Provenance information if available.
/// </summary>
public BundleProvenance? Provenance { get; init; }
}
/// <summary>
/// Provenance information for a bundle.
/// </summary>
public sealed record BundleProvenance
{
/// <summary>
/// Origin of the bundle data.
/// </summary>
public required string Origin { get; init; }
/// <summary>
/// Signature information if signed.
/// </summary>
public BundleSignature? Signature { get; init; }
/// <summary>
/// When the bundle was retrieved.
/// </summary>
public DateTimeOffset RetrievedAt { get; init; }
/// <summary>
/// Pipeline version that created this bundle.
/// </summary>
public string? PipelineVersion { get; init; }
}
/// <summary>
/// Signature information for a bundle.
/// </summary>
public sealed record BundleSignature
{
/// <summary>
/// Signature format (dsse, pgp, etc.).
/// </summary>
public required string Format { get; init; }
/// <summary>
/// Key identifier.
/// </summary>
public required string KeyId { get; init; }
/// <summary>
/// Whether signature was verified.
/// </summary>
public bool Verified { get; init; }
/// <summary>
/// When signature was verified.
/// </summary>
public DateTimeOffset? VerifiedAt { get; init; }
}

View File

@@ -0,0 +1,96 @@
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.AirGap.Models;
/// <summary>
/// Information about a registered bundle source.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed record BundleSourceInfo
{
/// <summary>
/// Unique identifier for the source.
/// </summary>
public required string Id { get; init; }
/// <summary>
/// Display name for the source.
/// </summary>
public string? DisplayName { get; init; }
/// <summary>
/// Source type (directory, archive, remote).
/// </summary>
public required string Type { get; init; }
/// <summary>
/// Location of the source (path or URL).
/// </summary>
public required string Location { get; init; }
/// <summary>
/// Whether the source is enabled.
/// </summary>
public bool Enabled { get; init; } = true;
/// <summary>
/// Priority for this source (lower = higher priority).
/// </summary>
public int Priority { get; init; } = 100;
/// <summary>
/// Verification mode for bundles (signature, hash, none).
/// </summary>
public string VerificationMode { get; init; } = "signature";
/// <summary>
/// When the source was registered.
/// </summary>
public DateTimeOffset RegisteredAt { get; init; }
/// <summary>
/// When the source was last validated.
/// </summary>
public DateTimeOffset? LastValidatedAt { get; init; }
/// <summary>
/// Number of bundles available from this source.
/// </summary>
public int BundleCount { get; init; }
/// <summary>
/// Source health status.
/// </summary>
public BundleSourceStatus Status { get; init; } = BundleSourceStatus.Unknown;
/// <summary>
/// Error message if status is Error.
/// </summary>
public string? ErrorMessage { get; init; }
/// <summary>
/// Metadata from the source catalog.
/// </summary>
public ImmutableDictionary<string, string> Metadata { get; init; } = ImmutableDictionary<string, string>.Empty;
}
/// <summary>
/// Bundle source health status.
/// </summary>
public enum BundleSourceStatus
{
/// <summary>Status unknown (not yet validated).</summary>
Unknown = 0,
/// <summary>Source is healthy and accessible.</summary>
Healthy = 1,
/// <summary>Source has warnings but is functional.</summary>
Degraded = 2,
/// <summary>Source is in error state.</summary>
Error = 3,
/// <summary>Source is disabled.</summary>
Disabled = 4
}

View File

@@ -0,0 +1,43 @@
namespace StellaOps.Concelier.Core.AirGap.Models;
/// <summary>
/// Registration request for a new bundle source.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed record BundleSourceRegistration
{
/// <summary>
/// Unique identifier for the source.
/// </summary>
public required string Id { get; init; }
/// <summary>
/// Display name for the source.
/// </summary>
public string? DisplayName { get; init; }
/// <summary>
/// Source type (directory, archive, remote).
/// </summary>
public required string Type { get; init; }
/// <summary>
/// Location of the source (path or URL).
/// </summary>
public required string Location { get; init; }
/// <summary>
/// Whether the source should be enabled immediately.
/// </summary>
public bool Enabled { get; init; } = true;
/// <summary>
/// Priority for this source (lower = higher priority).
/// </summary>
public int Priority { get; init; } = 100;
/// <summary>
/// Verification mode for bundles (signature, hash, none).
/// </summary>
public string VerificationMode { get; init; } = "signature";
}

View File

@@ -0,0 +1,69 @@
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.AirGap.Models;
/// <summary>
/// Result of validating a bundle source.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed record BundleSourceValidationResult
{
/// <summary>
/// Source identifier that was validated.
/// </summary>
public required string SourceId { get; init; }
/// <summary>
/// Whether the source is valid.
/// </summary>
public bool IsValid { get; init; }
/// <summary>
/// Source status after validation.
/// </summary>
public BundleSourceStatus Status { get; init; }
/// <summary>
/// Validation errors if any.
/// </summary>
public ImmutableArray<string> Errors { get; init; } = ImmutableArray<string>.Empty;
/// <summary>
/// Validation warnings if any.
/// </summary>
public ImmutableArray<string> Warnings { get; init; } = ImmutableArray<string>.Empty;
/// <summary>
/// Number of bundles discovered.
/// </summary>
public int BundleCount { get; init; }
/// <summary>
/// When the validation was performed.
/// </summary>
public DateTimeOffset ValidatedAt { get; init; }
/// <summary>
/// Creates a successful validation result.
/// </summary>
public static BundleSourceValidationResult Success(string sourceId, int bundleCount) => new()
{
SourceId = sourceId,
IsValid = true,
Status = BundleSourceStatus.Healthy,
BundleCount = bundleCount,
ValidatedAt = DateTimeOffset.UtcNow
};
/// <summary>
/// Creates a failed validation result.
/// </summary>
public static BundleSourceValidationResult Failure(string sourceId, params string[] errors) => new()
{
SourceId = sourceId,
IsValid = false,
Status = BundleSourceStatus.Error,
Errors = errors.ToImmutableArray(),
ValidatedAt = DateTimeOffset.UtcNow
};
}

View File

@@ -0,0 +1,71 @@
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.AirGap.Models;
/// <summary>
/// Status of sealed mode enforcement.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed record SealedModeStatus
{
/// <summary>
/// Whether sealed mode is enabled.
/// </summary>
public bool IsSealed { get; init; }
/// <summary>
/// Whether warn-only mode is active.
/// </summary>
public bool WarnOnly { get; init; }
/// <summary>
/// Sources that are allowed even in sealed mode.
/// </summary>
public ImmutableArray<string> AllowedSources { get; init; } = ImmutableArray<string>.Empty;
/// <summary>
/// Hosts that are allowed even in sealed mode.
/// </summary>
public ImmutableArray<string> AllowedHosts { get; init; } = ImmutableArray<string>.Empty;
/// <summary>
/// Sources that are currently blocked.
/// </summary>
public ImmutableArray<string> BlockedSources { get; init; } = ImmutableArray<string>.Empty;
/// <summary>
/// Recent seal violations (if warn-only mode).
/// </summary>
public ImmutableArray<SealViolation> RecentViolations { get; init; } = ImmutableArray<SealViolation>.Empty;
/// <summary>
/// When status was computed.
/// </summary>
public DateTimeOffset ComputedAt { get; init; }
}
/// <summary>
/// Record of a seal mode violation attempt.
/// </summary>
public sealed record SealViolation
{
/// <summary>
/// Source that attempted the violation.
/// </summary>
public required string SourceName { get; init; }
/// <summary>
/// Destination that was blocked.
/// </summary>
public required string Destination { get; init; }
/// <summary>
/// When the violation occurred.
/// </summary>
public DateTimeOffset OccurredAt { get; init; }
/// <summary>
/// Whether the request was blocked or just warned.
/// </summary>
public bool WasBlocked { get; init; }
}

View File

@@ -0,0 +1,169 @@
using System.Collections.Concurrent;
using System.Collections.Immutable;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.AirGap.Models;
namespace StellaOps.Concelier.Core.AirGap;
/// <summary>
/// Default implementation of <see cref="ISealedModeEnforcer"/>.
/// Per CONCELIER-WEB-AIRGAP-56-001.
/// </summary>
public sealed class SealedModeEnforcer : ISealedModeEnforcer
{
private readonly ILogger<SealedModeEnforcer> _logger;
private readonly TimeProvider _timeProvider;
private readonly bool _isSealed;
private readonly bool _warnOnly;
private readonly ImmutableHashSet<string> _allowedSources;
private readonly ImmutableHashSet<string> _allowedHosts;
private readonly ConcurrentQueue<SealViolation> _recentViolations = new();
private readonly ConcurrentDictionary<string, bool> _blockedSources = new(StringComparer.OrdinalIgnoreCase);
private const int MaxRecentViolations = 100;
public SealedModeEnforcer(
ILogger<SealedModeEnforcer> logger,
bool isSealed = false,
bool warnOnly = false,
IEnumerable<string>? allowedSources = null,
IEnumerable<string>? allowedHosts = null,
TimeProvider? timeProvider = null)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_isSealed = isSealed;
_warnOnly = warnOnly;
_allowedSources = (allowedSources ?? Enumerable.Empty<string>())
.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase);
_allowedHosts = (allowedHosts ?? Enumerable.Empty<string>())
.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase);
}
/// <inheritdoc />
public bool IsSealed => _isSealed;
/// <inheritdoc />
public void EnsureSourceAllowed(string sourceName, Uri destination)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceName);
ArgumentNullException.ThrowIfNull(destination);
if (!_isSealed)
{
return;
}
if (IsAllowed(sourceName, destination))
{
return;
}
RecordViolation(sourceName, destination);
if (_warnOnly)
{
_logger.LogWarning(
"Sealed mode violation (warn-only): source '{SourceName}' attempted to access '{Destination}'",
sourceName, destination);
return;
}
_logger.LogError(
"Sealed mode violation blocked: source '{SourceName}' attempted to access '{Destination}'",
sourceName, destination);
throw new SealedModeViolationException(sourceName, destination);
}
/// <inheritdoc />
public bool IsSourceAllowed(string sourceName, Uri destination)
{
if (!_isSealed)
{
return true;
}
return IsAllowed(sourceName, destination);
}
/// <inheritdoc />
public IReadOnlyList<string> GetBlockedSources()
=> _blockedSources.Keys.ToList();
/// <inheritdoc />
public SealedModeStatus GetStatus()
{
var violations = new List<SealViolation>();
foreach (var v in _recentViolations)
{
violations.Add(v);
}
return new SealedModeStatus
{
IsSealed = _isSealed,
WarnOnly = _warnOnly,
AllowedSources = _allowedSources.ToImmutableArray(),
AllowedHosts = _allowedHosts.ToImmutableArray(),
BlockedSources = _blockedSources.Keys.ToImmutableArray(),
RecentViolations = violations.TakeLast(20).ToImmutableArray(),
ComputedAt = _timeProvider.GetUtcNow()
};
}
private bool IsAllowed(string sourceName, Uri destination)
{
// Check if source is explicitly allowed
if (_allowedSources.Contains(sourceName))
{
return true;
}
// Check if host is explicitly allowed
if (_allowedHosts.Contains(destination.Host))
{
return true;
}
// Check for localhost/internal addresses
if (IsLocalAddress(destination))
{
return true;
}
// Mark source as blocked for status reporting
_blockedSources.TryAdd(sourceName, true);
return false;
}
private static bool IsLocalAddress(Uri uri)
{
var host = uri.Host.ToLowerInvariant();
return host == "localhost" ||
host == "127.0.0.1" ||
host == "::1" ||
host.StartsWith("192.168.") ||
host.StartsWith("10.") ||
host.StartsWith("172.16.") ||
host.EndsWith(".local");
}
private void RecordViolation(string sourceName, Uri destination)
{
var violation = new SealViolation
{
SourceName = sourceName,
Destination = destination.ToString(),
OccurredAt = _timeProvider.GetUtcNow(),
WasBlocked = !_warnOnly
};
_recentViolations.Enqueue(violation);
// Trim old violations
while (_recentViolations.Count > MaxRecentViolations)
{
_recentViolations.TryDequeue(out _);
}
}
}

View File

@@ -0,0 +1,313 @@
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Service for migrating raw payloads from GridFS to S3-compatible object storage.
/// </summary>
public sealed class GridFsMigrationService
{
private readonly IGridFSBucket _gridFs;
private readonly IObjectStore _objectStore;
private readonly IMigrationTracker _migrationTracker;
private readonly ObjectStorageOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<GridFsMigrationService> _logger;
public GridFsMigrationService(
IGridFSBucket gridFs,
IObjectStore objectStore,
IMigrationTracker migrationTracker,
IOptions<ObjectStorageOptions> options,
TimeProvider timeProvider,
ILogger<GridFsMigrationService> logger)
{
_gridFs = gridFs ?? throw new ArgumentNullException(nameof(gridFs));
_objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore));
_migrationTracker = migrationTracker ?? throw new ArgumentNullException(nameof(migrationTracker));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <summary>
/// Migrates a single GridFS document to object storage.
/// </summary>
public async Task<MigrationResult> MigrateAsync(
string gridFsId,
string tenantId,
string sourceId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(gridFsId);
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
// Check if already migrated
if (await _migrationTracker.IsMigratedAsync(gridFsId, cancellationToken).ConfigureAwait(false))
{
_logger.LogDebug("GridFS {GridFsId} already migrated, skipping", gridFsId);
return MigrationResult.AlreadyMigrated(gridFsId);
}
try
{
// Download from GridFS
var objectId = ObjectId.Parse(gridFsId);
using var downloadStream = new MemoryStream();
await _gridFs.DownloadToStreamAsync(objectId, downloadStream, cancellationToken: cancellationToken)
.ConfigureAwait(false);
var data = downloadStream.ToArray();
var sha256 = ComputeSha256(data);
// Get GridFS file info
var filter = Builders<GridFSFileInfo>.Filter.Eq("_id", objectId);
var fileInfo = await _gridFs.Find(filter)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
var ingestedAt = fileInfo?.UploadDateTime ?? _timeProvider.GetUtcNow().UtcDateTime;
// Create provenance metadata
var provenance = new ProvenanceMetadata
{
SourceId = sourceId,
IngestedAt = new DateTimeOffset(ingestedAt, TimeSpan.Zero),
TenantId = tenantId,
OriginalFormat = DetectFormat(fileInfo?.Filename),
OriginalSize = data.Length,
GridFsLegacyId = gridFsId,
Transformations =
[
new TransformationRecord
{
Type = TransformationType.Migration,
Timestamp = _timeProvider.GetUtcNow(),
Agent = "concelier-gridfs-migration-v1"
}
]
};
// Store in object storage
var reference = await _objectStore.StoreAsync(
tenantId,
data,
provenance,
GetContentType(fileInfo?.Filename),
cancellationToken).ConfigureAwait(false);
// Record migration
await _migrationTracker.RecordMigrationAsync(
gridFsId,
reference.Pointer,
MigrationStatus.Migrated,
cancellationToken).ConfigureAwait(false);
_logger.LogInformation(
"Migrated GridFS {GridFsId} to {Bucket}/{Key}, size {Size} bytes",
gridFsId, reference.Pointer.Bucket, reference.Pointer.Key, data.Length);
return MigrationResult.Success(gridFsId, reference);
}
catch (GridFSFileNotFoundException)
{
_logger.LogWarning("GridFS file not found: {GridFsId}", gridFsId);
return MigrationResult.NotFound(gridFsId);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to migrate GridFS {GridFsId}", gridFsId);
return MigrationResult.Failed(gridFsId, ex.Message);
}
}
/// <summary>
/// Verifies a migrated document by comparing hashes.
/// </summary>
public async Task<bool> VerifyMigrationAsync(
string gridFsId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(gridFsId);
var record = await _migrationTracker.GetByGridFsIdAsync(gridFsId, cancellationToken)
.ConfigureAwait(false);
if (record is null)
{
_logger.LogWarning("No migration record found for {GridFsId}", gridFsId);
return false;
}
// Download original from GridFS
var objectId = ObjectId.Parse(gridFsId);
using var downloadStream = new MemoryStream();
try
{
await _gridFs.DownloadToStreamAsync(objectId, downloadStream, cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
catch (GridFSFileNotFoundException)
{
_logger.LogWarning("Original GridFS file not found for verification: {GridFsId}", gridFsId);
return false;
}
var originalHash = ComputeSha256(downloadStream.ToArray());
// Verify the migrated object
var reference = PayloadReference.CreateObjectStorage(record.Pointer, new ProvenanceMetadata
{
SourceId = string.Empty,
IngestedAt = record.MigratedAt,
TenantId = string.Empty,
});
var verified = await _objectStore.VerifyIntegrityAsync(reference, cancellationToken)
.ConfigureAwait(false);
if (verified && string.Equals(originalHash, record.Pointer.Sha256, StringComparison.OrdinalIgnoreCase))
{
await _migrationTracker.MarkVerifiedAsync(gridFsId, cancellationToken).ConfigureAwait(false);
_logger.LogInformation("Verified migration for {GridFsId}", gridFsId);
return true;
}
_logger.LogWarning(
"Verification failed for {GridFsId}: original hash {Original}, stored hash {Stored}",
gridFsId, originalHash, record.Pointer.Sha256);
return false;
}
/// <summary>
/// Batches migration of multiple GridFS documents.
/// </summary>
public async Task<BatchMigrationResult> MigrateBatchAsync(
IEnumerable<GridFsMigrationRequest> requests,
CancellationToken cancellationToken = default)
{
var results = new List<MigrationResult>();
foreach (var request in requests)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var result = await MigrateAsync(
request.GridFsId,
request.TenantId,
request.SourceId,
cancellationToken).ConfigureAwait(false);
results.Add(result);
}
return new BatchMigrationResult(results);
}
private static string ComputeSha256(byte[] data)
{
var hash = SHA256.HashData(data);
return Convert.ToHexStringLower(hash);
}
private static OriginalFormat? DetectFormat(string? filename)
{
if (string.IsNullOrEmpty(filename))
{
return null;
}
return Path.GetExtension(filename).ToLowerInvariant() switch
{
".json" => OriginalFormat.Json,
".xml" => OriginalFormat.Xml,
".csv" => OriginalFormat.Csv,
".ndjson" => OriginalFormat.Ndjson,
".yaml" or ".yml" => OriginalFormat.Yaml,
_ => null
};
}
private static string GetContentType(string? filename)
{
if (string.IsNullOrEmpty(filename))
{
return "application/octet-stream";
}
return Path.GetExtension(filename).ToLowerInvariant() switch
{
".json" => "application/json",
".xml" => "application/xml",
".csv" => "text/csv",
".ndjson" => "application/x-ndjson",
".yaml" or ".yml" => "application/x-yaml",
_ => "application/octet-stream"
};
}
}
/// <summary>
/// Request to migrate a GridFS document.
/// </summary>
public sealed record GridFsMigrationRequest(
string GridFsId,
string TenantId,
string SourceId);
/// <summary>
/// Result of a single migration.
/// </summary>
public sealed record MigrationResult
{
public required string GridFsId { get; init; }
public required MigrationResultStatus Status { get; init; }
public PayloadReference? Reference { get; init; }
public string? ErrorMessage { get; init; }
public static MigrationResult Success(string gridFsId, PayloadReference reference)
=> new() { GridFsId = gridFsId, Status = MigrationResultStatus.Success, Reference = reference };
public static MigrationResult AlreadyMigrated(string gridFsId)
=> new() { GridFsId = gridFsId, Status = MigrationResultStatus.AlreadyMigrated };
public static MigrationResult NotFound(string gridFsId)
=> new() { GridFsId = gridFsId, Status = MigrationResultStatus.NotFound };
public static MigrationResult Failed(string gridFsId, string errorMessage)
=> new() { GridFsId = gridFsId, Status = MigrationResultStatus.Failed, ErrorMessage = errorMessage };
}
/// <summary>
/// Status of a migration result.
/// </summary>
public enum MigrationResultStatus
{
Success,
AlreadyMigrated,
NotFound,
Failed
}
/// <summary>
/// Result of a batch migration.
/// </summary>
public sealed record BatchMigrationResult(IReadOnlyList<MigrationResult> Results)
{
public int TotalCount => Results.Count;
public int SuccessCount => Results.Count(r => r.Status == MigrationResultStatus.Success);
public int AlreadyMigratedCount => Results.Count(r => r.Status == MigrationResultStatus.AlreadyMigrated);
public int NotFoundCount => Results.Count(r => r.Status == MigrationResultStatus.NotFound);
public int FailedCount => Results.Count(r => r.Status == MigrationResultStatus.Failed);
}

View File

@@ -0,0 +1,60 @@
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Tracks GridFS to S3 migrations.
/// </summary>
public interface IMigrationTracker
{
/// <summary>
/// Records a migration attempt.
/// </summary>
Task<MigrationRecord> RecordMigrationAsync(
string gridFsId,
ObjectPointer pointer,
MigrationStatus status,
CancellationToken cancellationToken = default);
/// <summary>
/// Updates a migration record status.
/// </summary>
Task UpdateStatusAsync(
string gridFsId,
MigrationStatus status,
string? errorMessage = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Marks a migration as verified.
/// </summary>
Task MarkVerifiedAsync(
string gridFsId,
CancellationToken cancellationToken = default);
/// <summary>
/// Gets a migration record by GridFS ID.
/// </summary>
Task<MigrationRecord?> GetByGridFsIdAsync(
string gridFsId,
CancellationToken cancellationToken = default);
/// <summary>
/// Lists pending migrations.
/// </summary>
Task<IReadOnlyList<MigrationRecord>> ListPendingAsync(
int limit = 100,
CancellationToken cancellationToken = default);
/// <summary>
/// Lists migrations needing verification.
/// </summary>
Task<IReadOnlyList<MigrationRecord>> ListNeedingVerificationAsync(
int limit = 100,
CancellationToken cancellationToken = default);
/// <summary>
/// Checks if a GridFS ID has been migrated.
/// </summary>
Task<bool> IsMigratedAsync(
string gridFsId,
CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,98 @@
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Abstraction for S3-compatible object storage operations.
/// </summary>
public interface IObjectStore
{
/// <summary>
/// Stores a payload, returning a reference (either inline or object storage).
/// Automatically decides based on size thresholds.
/// </summary>
/// <param name="tenantId">Tenant identifier for bucket selection.</param>
/// <param name="data">Payload data to store.</param>
/// <param name="provenance">Provenance metadata for the payload.</param>
/// <param name="contentType">MIME type of the content.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Reference to the stored payload.</returns>
Task<PayloadReference> StoreAsync(
string tenantId,
ReadOnlyMemory<byte> data,
ProvenanceMetadata provenance,
string contentType = "application/json",
CancellationToken cancellationToken = default);
/// <summary>
/// Stores a payload from a stream.
/// </summary>
/// <param name="tenantId">Tenant identifier for bucket selection.</param>
/// <param name="stream">Stream containing payload data.</param>
/// <param name="provenance">Provenance metadata for the payload.</param>
/// <param name="contentType">MIME type of the content.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Reference to the stored payload.</returns>
Task<PayloadReference> StoreStreamAsync(
string tenantId,
Stream stream,
ProvenanceMetadata provenance,
string contentType = "application/json",
CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves a payload by its reference.
/// </summary>
/// <param name="reference">Reference to the payload.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Payload data, or null if not found.</returns>
Task<byte[]?> RetrieveAsync(
PayloadReference reference,
CancellationToken cancellationToken = default);
/// <summary>
/// Retrieves a payload as a stream.
/// </summary>
/// <param name="reference">Reference to the payload.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Stream containing payload data, or null if not found.</returns>
Task<Stream?> RetrieveStreamAsync(
PayloadReference reference,
CancellationToken cancellationToken = default);
/// <summary>
/// Checks if an object exists.
/// </summary>
/// <param name="pointer">Object pointer to check.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>True if object exists.</returns>
Task<bool> ExistsAsync(
ObjectPointer pointer,
CancellationToken cancellationToken = default);
/// <summary>
/// Deletes an object.
/// </summary>
/// <param name="pointer">Object pointer to delete.</param>
/// <param name="cancellationToken">Cancellation token.</param>
Task DeleteAsync(
ObjectPointer pointer,
CancellationToken cancellationToken = default);
/// <summary>
/// Ensures the tenant bucket exists.
/// </summary>
/// <param name="tenantId">Tenant identifier.</param>
/// <param name="cancellationToken">Cancellation token.</param>
Task EnsureBucketExistsAsync(
string tenantId,
CancellationToken cancellationToken = default);
/// <summary>
/// Verifies a payload's integrity by comparing its hash.
/// </summary>
/// <param name="reference">Reference to verify.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>True if hash matches.</returns>
Task<bool> VerifyIntegrityAsync(
PayloadReference reference,
CancellationToken cancellationToken = default);
}

View File

@@ -0,0 +1,63 @@
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Record of a migration from GridFS to S3.
/// </summary>
public sealed record MigrationRecord
{
/// <summary>
/// Original GridFS ObjectId.
/// </summary>
public required string GridFsId { get; init; }
/// <summary>
/// Pointer to the migrated object.
/// </summary>
public required ObjectPointer Pointer { get; init; }
/// <summary>
/// Timestamp when migration was performed.
/// </summary>
public required DateTimeOffset MigratedAt { get; init; }
/// <summary>
/// Current status of the migration.
/// </summary>
public required MigrationStatus Status { get; init; }
/// <summary>
/// Timestamp when content hash was verified post-migration.
/// </summary>
public DateTimeOffset? VerifiedAt { get; init; }
/// <summary>
/// Whether GridFS tombstone still exists for rollback.
/// </summary>
public bool RollbackAvailable { get; init; } = true;
/// <summary>
/// Error message if migration failed.
/// </summary>
public string? ErrorMessage { get; init; }
}
/// <summary>
/// Status of a GridFS to S3 migration.
/// </summary>
public enum MigrationStatus
{
/// <summary>Migration pending.</summary>
Pending,
/// <summary>Migration completed.</summary>
Migrated,
/// <summary>Migration verified via hash comparison.</summary>
Verified,
/// <summary>Migration failed.</summary>
Failed,
/// <summary>Original GridFS tombstoned.</summary>
Tombstoned
}

View File

@@ -0,0 +1,232 @@
using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// MongoDB-backed migration tracker for GridFS to S3 migrations.
/// </summary>
public sealed class MongoMigrationTracker : IMigrationTracker
{
private const string CollectionName = "object_storage_migrations";
private readonly IMongoCollection<MigrationDocument> _collection;
private readonly TimeProvider _timeProvider;
private readonly ILogger<MongoMigrationTracker> _logger;
public MongoMigrationTracker(
IMongoDatabase database,
TimeProvider timeProvider,
ILogger<MongoMigrationTracker> logger)
{
ArgumentNullException.ThrowIfNull(database);
_collection = database.GetCollection<MigrationDocument>(CollectionName);
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<MigrationRecord> RecordMigrationAsync(
string gridFsId,
ObjectPointer pointer,
MigrationStatus status,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(gridFsId);
ArgumentNullException.ThrowIfNull(pointer);
var now = _timeProvider.GetUtcNow();
var document = new MigrationDocument
{
GridFsId = gridFsId,
Bucket = pointer.Bucket,
Key = pointer.Key,
Sha256 = pointer.Sha256,
Size = pointer.Size,
ContentType = pointer.ContentType,
Encoding = pointer.Encoding.ToString().ToLowerInvariant(),
MigratedAt = now.UtcDateTime,
Status = status.ToString().ToLowerInvariant(),
RollbackAvailable = true,
};
await _collection.InsertOneAsync(document, cancellationToken: cancellationToken)
.ConfigureAwait(false);
_logger.LogInformation(
"Recorded migration for GridFS {GridFsId} to {Bucket}/{Key}",
gridFsId, pointer.Bucket, pointer.Key);
return ToRecord(document);
}
public async Task UpdateStatusAsync(
string gridFsId,
MigrationStatus status,
string? errorMessage = null,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(gridFsId);
var filter = Builders<MigrationDocument>.Filter.Eq(d => d.GridFsId, gridFsId);
var update = Builders<MigrationDocument>.Update
.Set(d => d.Status, status.ToString().ToLowerInvariant())
.Set(d => d.ErrorMessage, errorMessage);
await _collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken)
.ConfigureAwait(false);
_logger.LogDebug("Updated migration status for {GridFsId} to {Status}", gridFsId, status);
}
public async Task MarkVerifiedAsync(
string gridFsId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(gridFsId);
var now = _timeProvider.GetUtcNow();
var filter = Builders<MigrationDocument>.Filter.Eq(d => d.GridFsId, gridFsId);
var update = Builders<MigrationDocument>.Update
.Set(d => d.Status, MigrationStatus.Verified.ToString().ToLowerInvariant())
.Set(d => d.VerifiedAt, now.UtcDateTime);
await _collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken)
.ConfigureAwait(false);
_logger.LogDebug("Marked migration as verified for {GridFsId}", gridFsId);
}
public async Task<MigrationRecord?> GetByGridFsIdAsync(
string gridFsId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(gridFsId);
var filter = Builders<MigrationDocument>.Filter.Eq(d => d.GridFsId, gridFsId);
var document = await _collection.Find(filter)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
return document is null ? null : ToRecord(document);
}
public async Task<IReadOnlyList<MigrationRecord>> ListPendingAsync(
int limit = 100,
CancellationToken cancellationToken = default)
{
var filter = Builders<MigrationDocument>.Filter.Eq(
d => d.Status, MigrationStatus.Pending.ToString().ToLowerInvariant());
var documents = await _collection.Find(filter)
.Limit(limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents.Select(ToRecord).ToList();
}
public async Task<IReadOnlyList<MigrationRecord>> ListNeedingVerificationAsync(
int limit = 100,
CancellationToken cancellationToken = default)
{
var filter = Builders<MigrationDocument>.Filter.Eq(
d => d.Status, MigrationStatus.Migrated.ToString().ToLowerInvariant());
var documents = await _collection.Find(filter)
.Limit(limit)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
return documents.Select(ToRecord).ToList();
}
public async Task<bool> IsMigratedAsync(
string gridFsId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(gridFsId);
var filter = Builders<MigrationDocument>.Filter.And(
Builders<MigrationDocument>.Filter.Eq(d => d.GridFsId, gridFsId),
Builders<MigrationDocument>.Filter.In(d => d.Status, new[]
{
MigrationStatus.Migrated.ToString().ToLowerInvariant(),
MigrationStatus.Verified.ToString().ToLowerInvariant()
}));
var count = await _collection.CountDocumentsAsync(filter, cancellationToken: cancellationToken)
.ConfigureAwait(false);
return count > 0;
}
private static MigrationRecord ToRecord(MigrationDocument document)
{
return new MigrationRecord
{
GridFsId = document.GridFsId,
Pointer = new ObjectPointer
{
Bucket = document.Bucket,
Key = document.Key,
Sha256 = document.Sha256,
Size = document.Size,
ContentType = document.ContentType,
Encoding = Enum.Parse<ContentEncoding>(document.Encoding, ignoreCase: true),
},
MigratedAt = new DateTimeOffset(document.MigratedAt, TimeSpan.Zero),
Status = Enum.Parse<MigrationStatus>(document.Status, ignoreCase: true),
VerifiedAt = document.VerifiedAt.HasValue
? new DateTimeOffset(document.VerifiedAt.Value, TimeSpan.Zero)
: null,
RollbackAvailable = document.RollbackAvailable,
ErrorMessage = document.ErrorMessage,
};
}
[BsonIgnoreExtraElements]
private sealed class MigrationDocument
{
[BsonId]
[BsonRepresentation(BsonType.ObjectId)]
public string? Id { get; set; }
[BsonElement("gridFsId")]
public required string GridFsId { get; set; }
[BsonElement("bucket")]
public required string Bucket { get; set; }
[BsonElement("key")]
public required string Key { get; set; }
[BsonElement("sha256")]
public required string Sha256 { get; set; }
[BsonElement("size")]
public required long Size { get; set; }
[BsonElement("contentType")]
public required string ContentType { get; set; }
[BsonElement("encoding")]
public required string Encoding { get; set; }
[BsonElement("migratedAt")]
public required DateTime MigratedAt { get; set; }
[BsonElement("status")]
public required string Status { get; set; }
[BsonElement("verifiedAt")]
public DateTime? VerifiedAt { get; set; }
[BsonElement("rollbackAvailable")]
public bool RollbackAvailable { get; set; }
[BsonElement("errorMessage")]
public string? ErrorMessage { get; set; }
}
}

View File

@@ -0,0 +1,52 @@
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Deterministic pointer to an object in S3-compatible storage.
/// </summary>
public sealed record ObjectPointer
{
/// <summary>
/// S3 bucket name (tenant-prefixed).
/// </summary>
public required string Bucket { get; init; }
/// <summary>
/// Object key (deterministic, content-addressed).
/// </summary>
public required string Key { get; init; }
/// <summary>
/// SHA-256 hash of object content (hex encoded).
/// </summary>
public required string Sha256 { get; init; }
/// <summary>
/// Object size in bytes.
/// </summary>
public required long Size { get; init; }
/// <summary>
/// MIME type of the object.
/// </summary>
public string ContentType { get; init; } = "application/octet-stream";
/// <summary>
/// Content encoding if compressed.
/// </summary>
public ContentEncoding Encoding { get; init; } = ContentEncoding.Identity;
}
/// <summary>
/// Content encoding for stored objects.
/// </summary>
public enum ContentEncoding
{
/// <summary>No compression.</summary>
Identity,
/// <summary>Gzip compression.</summary>
Gzip,
/// <summary>Zstandard compression.</summary>
Zstd
}

View File

@@ -0,0 +1,75 @@
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Configuration options for S3-compatible object storage.
/// </summary>
public sealed class ObjectStorageOptions
{
/// <summary>
/// Configuration section name.
/// </summary>
public const string SectionName = "Concelier:ObjectStorage";
/// <summary>
/// S3-compatible endpoint URL (MinIO, AWS S3, etc.).
/// </summary>
public string Endpoint { get; set; } = "http://localhost:9000";
/// <summary>
/// Storage region (use 'us-east-1' for MinIO).
/// </summary>
public string Region { get; set; } = "us-east-1";
/// <summary>
/// Use path-style addressing (required for MinIO).
/// </summary>
public bool UsePathStyle { get; set; } = true;
/// <summary>
/// Prefix for tenant bucket names.
/// </summary>
public string BucketPrefix { get; set; } = "stellaops-concelier-";
/// <summary>
/// Maximum object size in bytes (default 5GB).
/// </summary>
public long MaxObjectSize { get; set; } = 5L * 1024 * 1024 * 1024;
/// <summary>
/// Objects larger than this (bytes) will be compressed.
/// Default: 1MB.
/// </summary>
public int CompressionThreshold { get; set; } = 1024 * 1024;
/// <summary>
/// Objects smaller than this (bytes) will be stored inline.
/// Default: 64KB.
/// </summary>
public int InlineThreshold { get; set; } = 64 * 1024;
/// <summary>
/// Whether object storage is enabled. When false, uses GridFS fallback.
/// </summary>
public bool Enabled { get; set; } = false;
/// <summary>
/// AWS access key ID (or MinIO access key).
/// </summary>
public string? AccessKeyId { get; set; }
/// <summary>
/// AWS secret access key (or MinIO secret key).
/// </summary>
public string? SecretAccessKey { get; set; }
/// <summary>
/// Gets the bucket name for a tenant.
/// </summary>
public string GetBucketName(string tenantId)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
// Normalize tenant ID to lowercase and replace invalid characters
var normalized = tenantId.ToLowerInvariant().Replace('_', '-');
return $"{BucketPrefix}{normalized}";
}
}

View File

@@ -0,0 +1,128 @@
using Amazon;
using Amazon.Runtime;
using Amazon.S3;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Options;
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Extension methods for registering object storage services.
/// </summary>
public static class ObjectStorageServiceCollectionExtensions
{
/// <summary>
/// Adds object storage services for Concelier raw payload storage.
/// </summary>
public static IServiceCollection AddConcelierObjectStorage(
this IServiceCollection services,
IConfiguration configuration)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configuration);
// Bind options
services.Configure<ObjectStorageOptions>(
configuration.GetSection(ObjectStorageOptions.SectionName));
// Register TimeProvider if not already registered
services.TryAddSingleton(TimeProvider.System);
// Register S3 client
services.TryAddSingleton<IAmazonS3>(sp =>
{
var options = sp.GetRequiredService<IOptions<ObjectStorageOptions>>().Value;
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
ForcePathStyle = options.UsePathStyle,
};
if (!string.IsNullOrEmpty(options.Endpoint))
{
config.ServiceURL = options.Endpoint;
}
if (!string.IsNullOrEmpty(options.AccessKeyId) &&
!string.IsNullOrEmpty(options.SecretAccessKey))
{
var credentials = new BasicAWSCredentials(
options.AccessKeyId,
options.SecretAccessKey);
return new AmazonS3Client(credentials, config);
}
// Use default credentials chain (env vars, IAM role, etc.)
return new AmazonS3Client(config);
});
// Register object store
services.TryAddSingleton<IObjectStore, S3ObjectStore>();
// Register migration tracker
services.TryAddSingleton<IMigrationTracker, MongoMigrationTracker>();
// Register migration service
services.TryAddSingleton<GridFsMigrationService>();
return services;
}
/// <summary>
/// Adds object storage services with explicit options.
/// </summary>
public static IServiceCollection AddConcelierObjectStorage(
this IServiceCollection services,
Action<ObjectStorageOptions> configureOptions)
{
ArgumentNullException.ThrowIfNull(services);
ArgumentNullException.ThrowIfNull(configureOptions);
services.Configure(configureOptions);
// Register TimeProvider if not already registered
services.TryAddSingleton(TimeProvider.System);
// Register S3 client
services.TryAddSingleton<IAmazonS3>(sp =>
{
var options = sp.GetRequiredService<IOptions<ObjectStorageOptions>>().Value;
var config = new AmazonS3Config
{
RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
ForcePathStyle = options.UsePathStyle,
};
if (!string.IsNullOrEmpty(options.Endpoint))
{
config.ServiceURL = options.Endpoint;
}
if (!string.IsNullOrEmpty(options.AccessKeyId) &&
!string.IsNullOrEmpty(options.SecretAccessKey))
{
var credentials = new BasicAWSCredentials(
options.AccessKeyId,
options.SecretAccessKey);
return new AmazonS3Client(credentials, config);
}
return new AmazonS3Client(config);
});
// Register object store
services.TryAddSingleton<IObjectStore, S3ObjectStore>();
// Register migration tracker
services.TryAddSingleton<IMigrationTracker, MongoMigrationTracker>();
// Register migration service
services.TryAddSingleton<GridFsMigrationService>();
return services;
}
}

View File

@@ -0,0 +1,79 @@
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Reference to a large payload stored in object storage (used in advisory_observations).
/// </summary>
public sealed record PayloadReference
{
/// <summary>
/// Discriminator for payload type.
/// </summary>
public const string TypeDiscriminator = "object-storage-ref";
/// <summary>
/// Type discriminator value.
/// </summary>
public string Type { get; init; } = TypeDiscriminator;
/// <summary>
/// Pointer to the object in storage.
/// </summary>
public required ObjectPointer Pointer { get; init; }
/// <summary>
/// Provenance metadata for the payload.
/// </summary>
public required ProvenanceMetadata Provenance { get; init; }
/// <summary>
/// If true, payload is small enough to be inline (not in object storage).
/// </summary>
public bool Inline { get; init; }
/// <summary>
/// Base64-encoded inline data (only if Inline=true and size less than threshold).
/// </summary>
public string? InlineData { get; init; }
/// <summary>
/// Creates a reference for inline data.
/// </summary>
public static PayloadReference CreateInline(
byte[] data,
string sha256,
ProvenanceMetadata provenance,
string contentType = "application/octet-stream")
{
return new PayloadReference
{
Pointer = new ObjectPointer
{
Bucket = string.Empty,
Key = string.Empty,
Sha256 = sha256,
Size = data.Length,
ContentType = contentType,
Encoding = ContentEncoding.Identity,
},
Provenance = provenance,
Inline = true,
InlineData = Convert.ToBase64String(data),
};
}
/// <summary>
/// Creates a reference for object storage data.
/// </summary>
public static PayloadReference CreateObjectStorage(
ObjectPointer pointer,
ProvenanceMetadata provenance)
{
return new PayloadReference
{
Pointer = pointer,
Provenance = provenance,
Inline = false,
InlineData = null,
};
}
}

View File

@@ -0,0 +1,86 @@
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// Provenance metadata preserved from original ingestion.
/// </summary>
public sealed record ProvenanceMetadata
{
/// <summary>
/// Identifier of the original data source (URI).
/// </summary>
public required string SourceId { get; init; }
/// <summary>
/// UTC timestamp of original ingestion.
/// </summary>
public required DateTimeOffset IngestedAt { get; init; }
/// <summary>
/// Tenant identifier for multi-tenant isolation.
/// </summary>
public required string TenantId { get; init; }
/// <summary>
/// Original format before normalization.
/// </summary>
public OriginalFormat? OriginalFormat { get; init; }
/// <summary>
/// Original size before any transformation.
/// </summary>
public long? OriginalSize { get; init; }
/// <summary>
/// List of transformations applied.
/// </summary>
public IReadOnlyList<TransformationRecord> Transformations { get; init; } = [];
/// <summary>
/// Original GridFS ObjectId for migration tracking.
/// </summary>
public string? GridFsLegacyId { get; init; }
}
/// <summary>
/// Original format of ingested data.
/// </summary>
public enum OriginalFormat
{
Json,
Xml,
Csv,
Ndjson,
Yaml
}
/// <summary>
/// Record of a transformation applied to the payload.
/// </summary>
public sealed record TransformationRecord
{
/// <summary>
/// Type of transformation.
/// </summary>
public required TransformationType Type { get; init; }
/// <summary>
/// Timestamp when transformation was applied.
/// </summary>
public required DateTimeOffset Timestamp { get; init; }
/// <summary>
/// Agent/service that performed the transformation.
/// </summary>
public required string Agent { get; init; }
}
/// <summary>
/// Types of transformations that can be applied.
/// </summary>
public enum TransformationType
{
Compression,
Normalization,
Redaction,
Migration
}

View File

@@ -0,0 +1,320 @@
using System.IO.Compression;
using System.Security.Cryptography;
using Amazon.S3;
using Amazon.S3.Model;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
namespace StellaOps.Concelier.Storage.Mongo.ObjectStorage;
/// <summary>
/// S3-compatible object store implementation for raw advisory payloads.
/// </summary>
public sealed class S3ObjectStore : IObjectStore
{
private readonly IAmazonS3 _s3;
private readonly ObjectStorageOptions _options;
private readonly TimeProvider _timeProvider;
private readonly ILogger<S3ObjectStore> _logger;
public S3ObjectStore(
IAmazonS3 s3,
IOptions<ObjectStorageOptions> options,
TimeProvider timeProvider,
ILogger<S3ObjectStore> logger)
{
_s3 = s3 ?? throw new ArgumentNullException(nameof(s3));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_timeProvider = timeProvider ?? TimeProvider.System;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task<PayloadReference> StoreAsync(
string tenantId,
ReadOnlyMemory<byte> data,
ProvenanceMetadata provenance,
string contentType = "application/json",
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentNullException.ThrowIfNull(provenance);
var dataArray = data.ToArray();
var sha256 = ComputeSha256(dataArray);
// Use inline storage for small payloads
if (dataArray.Length < _options.InlineThreshold)
{
_logger.LogDebug(
"Storing inline payload for tenant {TenantId}, size {Size} bytes",
tenantId, dataArray.Length);
return PayloadReference.CreateInline(dataArray, sha256, provenance, contentType);
}
// Store in S3
var bucket = _options.GetBucketName(tenantId);
await EnsureBucketExistsAsync(tenantId, cancellationToken).ConfigureAwait(false);
var shouldCompress = dataArray.Length >= _options.CompressionThreshold;
var encoding = ContentEncoding.Identity;
byte[] payloadToStore = dataArray;
if (shouldCompress)
{
payloadToStore = CompressGzip(dataArray);
encoding = ContentEncoding.Gzip;
_logger.LogDebug(
"Compressed payload from {OriginalSize} to {CompressedSize} bytes",
dataArray.Length, payloadToStore.Length);
}
var key = GenerateKey(sha256, provenance.IngestedAt, contentType, encoding);
var request = new PutObjectRequest
{
BucketName = bucket,
Key = key,
InputStream = new MemoryStream(payloadToStore),
ContentType = encoding == ContentEncoding.Gzip ? "application/gzip" : contentType,
AutoCloseStream = true,
};
// Add metadata
request.Metadata["x-stellaops-sha256"] = sha256;
request.Metadata["x-stellaops-original-size"] = dataArray.Length.ToString();
request.Metadata["x-stellaops-encoding"] = encoding.ToString().ToLowerInvariant();
request.Metadata["x-stellaops-source-id"] = provenance.SourceId;
request.Metadata["x-stellaops-ingested-at"] = provenance.IngestedAt.ToString("O");
await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false);
_logger.LogDebug(
"Stored object {Bucket}/{Key}, size {Size} bytes, encoding {Encoding}",
bucket, key, payloadToStore.Length, encoding);
var pointer = new ObjectPointer
{
Bucket = bucket,
Key = key,
Sha256 = sha256,
Size = payloadToStore.Length,
ContentType = contentType,
Encoding = encoding,
};
return PayloadReference.CreateObjectStorage(pointer, provenance);
}
public async Task<PayloadReference> StoreStreamAsync(
string tenantId,
Stream stream,
ProvenanceMetadata provenance,
string contentType = "application/json",
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
ArgumentNullException.ThrowIfNull(stream);
ArgumentNullException.ThrowIfNull(provenance);
// Read stream to memory for hash computation
using var memoryStream = new MemoryStream();
await stream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false);
var data = memoryStream.ToArray();
return await StoreAsync(tenantId, data, provenance, contentType, cancellationToken)
.ConfigureAwait(false);
}
public async Task<byte[]?> RetrieveAsync(
PayloadReference reference,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(reference);
// Handle inline data
if (reference.Inline && reference.InlineData is not null)
{
return Convert.FromBase64String(reference.InlineData);
}
var stream = await RetrieveStreamAsync(reference, cancellationToken).ConfigureAwait(false);
if (stream is null)
{
return null;
}
using (stream)
{
using var memoryStream = new MemoryStream();
await stream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false);
return memoryStream.ToArray();
}
}
public async Task<Stream?> RetrieveStreamAsync(
PayloadReference reference,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(reference);
// Handle inline data
if (reference.Inline && reference.InlineData is not null)
{
return new MemoryStream(Convert.FromBase64String(reference.InlineData));
}
var pointer = reference.Pointer;
try
{
var response = await _s3.GetObjectAsync(pointer.Bucket, pointer.Key, cancellationToken)
.ConfigureAwait(false);
Stream resultStream = response.ResponseStream;
// Decompress if needed
if (pointer.Encoding == ContentEncoding.Gzip)
{
var decompressed = new MemoryStream();
using (var gzip = new GZipStream(response.ResponseStream, CompressionMode.Decompress))
{
await gzip.CopyToAsync(decompressed, cancellationToken).ConfigureAwait(false);
}
decompressed.Position = 0;
resultStream = decompressed;
}
return resultStream;
}
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.LogWarning("Object not found: {Bucket}/{Key}", pointer.Bucket, pointer.Key);
return null;
}
}
public async Task<bool> ExistsAsync(
ObjectPointer pointer,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(pointer);
try
{
var metadata = await _s3.GetObjectMetadataAsync(pointer.Bucket, pointer.Key, cancellationToken)
.ConfigureAwait(false);
return metadata.HttpStatusCode == System.Net.HttpStatusCode.OK;
}
catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
return false;
}
}
public async Task DeleteAsync(
ObjectPointer pointer,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(pointer);
await _s3.DeleteObjectAsync(pointer.Bucket, pointer.Key, cancellationToken)
.ConfigureAwait(false);
_logger.LogDebug("Deleted object {Bucket}/{Key}", pointer.Bucket, pointer.Key);
}
public async Task EnsureBucketExistsAsync(
string tenantId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tenantId);
var bucket = _options.GetBucketName(tenantId);
try
{
await _s3.EnsureBucketExistsAsync(bucket).ConfigureAwait(false);
_logger.LogDebug("Ensured bucket exists: {Bucket}", bucket);
}
catch (AmazonS3Exception ex)
{
_logger.LogError(ex, "Failed to ensure bucket exists: {Bucket}", bucket);
throw;
}
}
public async Task<bool> VerifyIntegrityAsync(
PayloadReference reference,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(reference);
var data = await RetrieveAsync(reference, cancellationToken).ConfigureAwait(false);
if (data is null)
{
return false;
}
var computedHash = ComputeSha256(data);
var matches = string.Equals(computedHash, reference.Pointer.Sha256, StringComparison.OrdinalIgnoreCase);
if (!matches)
{
_logger.LogWarning(
"Integrity check failed for {Bucket}/{Key}: expected {Expected}, got {Actual}",
reference.Pointer.Bucket, reference.Pointer.Key,
reference.Pointer.Sha256, computedHash);
}
return matches;
}
private static string ComputeSha256(byte[] data)
{
var hash = SHA256.HashData(data);
return Convert.ToHexStringLower(hash);
}
private static byte[] CompressGzip(byte[] data)
{
using var output = new MemoryStream();
using (var gzip = new GZipStream(output, CompressionLevel.Optimal, leaveOpen: true))
{
gzip.Write(data);
}
return output.ToArray();
}
private static string GenerateKey(
string sha256,
DateTimeOffset ingestedAt,
string contentType,
ContentEncoding encoding)
{
var date = ingestedAt.UtcDateTime;
var extension = GetExtension(contentType, encoding);
// Format: advisories/raw/YYYY/MM/DD/sha256-{hash}.{extension}
return $"advisories/raw/{date:yyyy}/{date:MM}/{date:dd}/sha256-{sha256[..16]}{extension}";
}
private static string GetExtension(string contentType, ContentEncoding encoding)
{
var baseExt = contentType switch
{
"application/json" => ".json",
"application/xml" or "text/xml" => ".xml",
"text/csv" => ".csv",
"application/x-ndjson" => ".ndjson",
"application/x-yaml" or "text/yaml" => ".yaml",
_ => ".bin"
};
return encoding switch
{
ContentEncoding.Gzip => baseExt + ".gz",
ContentEncoding.Zstd => baseExt + ".zst",
_ => baseExt
};
}
}

View File

@@ -4,7 +4,18 @@
<LangVersion>preview</LangVersion>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.305.6" />
<PackageReference Include="MongoDB.Driver" Version="3.5.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="10.0.0" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../StellaOps.Concelier.RawModels/StellaOps.Concelier.RawModels.csproj" />
<ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />