diff --git a/src/Concelier/StellaOps.Concelier.Plugin.Unified/FeedPluginAdapterFactory.cs b/src/Concelier/StellaOps.Concelier.Plugin.Unified/FeedPluginAdapterFactory.cs index aca5fb174..a30995205 100644 --- a/src/Concelier/StellaOps.Concelier.Plugin.Unified/FeedPluginAdapterFactory.cs +++ b/src/Concelier/StellaOps.Concelier.Plugin.Unified/FeedPluginAdapterFactory.cs @@ -63,6 +63,7 @@ public sealed class FeedPluginAdapterFactory ["kaspersky"] = FeedType.Advisory, // Mirror/EPSS + ["stella-mirror"] = FeedType.Database, ["stellaops-mirror"] = FeedType.Database, ["epss"] = FeedType.EcosystemSpecific }; diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/InternalSetupSourceEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/InternalSetupSourceEndpointExtensions.cs new file mode 100644 index 000000000..01fcf9088 --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/InternalSetupSourceEndpointExtensions.cs @@ -0,0 +1,98 @@ +using Microsoft.AspNetCore.Mvc; +using StellaOps.Concelier.WebService.Services; + +namespace StellaOps.Concelier.WebService.Extensions; + +internal static class InternalSetupSourceEndpointExtensions +{ + private const string BootstrapHeaderName = "X-StellaOps-Bootstrap-Key"; + + public static void MapInternalSetupSourceEndpoints(this WebApplication app) + { + var group = app.MapGroup("/internal/setup/advisory-sources") + .WithTags("Internal Setup"); + + group.MapPost("/probe", async Task ( + HttpContext context, + IConfiguration configuration, + [FromBody] SetupAdvisorySourcesBootstrapRequest? request, + [FromServices] ConfiguredAdvisorySourceService service, + CancellationToken cancellationToken) => + { + var authorizationFailure = ValidateBootstrapRequest(context, configuration); + if (authorizationFailure is not null) + { + return authorizationFailure; + } + + try + { + var result = await service.ProbeSetupAsync(request?.ConfigValues, cancellationToken).ConfigureAwait(false); + return TypedResults.Ok(result); + } + catch (InvalidOperationException ex) + { + return TypedResults.BadRequest(new ProblemDetails + { + Title = "Invalid advisory source configuration", + Detail = ex.Message, + Status = StatusCodes.Status400BadRequest + }); + } + }); + + group.MapPost("/apply", async Task ( + HttpContext context, + IConfiguration configuration, + [FromBody] SetupAdvisorySourcesBootstrapRequest? request, + [FromServices] ConfiguredAdvisorySourceService service, + CancellationToken cancellationToken) => + { + var authorizationFailure = ValidateBootstrapRequest(context, configuration); + if (authorizationFailure is not null) + { + return authorizationFailure; + } + + try + { + var result = await service.ApplySetupAsync(request?.ConfigValues, cancellationToken).ConfigureAwait(false); + return TypedResults.Ok(result); + } + catch (InvalidOperationException ex) + { + return TypedResults.BadRequest(new ProblemDetails + { + Title = "Invalid advisory source configuration", + Detail = ex.Message, + Status = StatusCodes.Status400BadRequest + }); + } + }); + } + + private static IResult? ValidateBootstrapRequest(HttpContext context, IConfiguration configuration) + { + var expectedKey = FirstNonEmpty( + configuration["STELLAOPS_BOOTSTRAP_KEY"], + configuration["Authority:Bootstrap:ApiKey"], + configuration["Authority:BootstrapKey"], + configuration["STELLAOPS_AUTHORITY_AUTHORITY__BOOTSTRAP__APIKEY"]); + + if (string.IsNullOrWhiteSpace(expectedKey)) + { + return TypedResults.Problem( + title: "Bootstrap key unavailable", + detail: "No bootstrap key is configured for internal advisory source setup.", + statusCode: StatusCodes.Status503ServiceUnavailable); + } + + var providedKey = context.Request.Headers[BootstrapHeaderName].FirstOrDefault(); + return string.Equals(expectedKey, providedKey, StringComparison.Ordinal) + ? null + : TypedResults.Unauthorized(); + } + + private static string FirstNonEmpty(params string?[] values) => + values.FirstOrDefault(static value => !string.IsNullOrWhiteSpace(value)) ?? string.Empty; +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs index c927ce075..2d4ff4587 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs @@ -3,8 +3,11 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using StellaOps.Concelier.Core.Jobs; +using StellaOps.DependencyInjection; using System; using System.Collections.Generic; +using System.Linq; +using System.Reflection; namespace StellaOps.Concelier.WebService.Extensions; @@ -148,12 +151,14 @@ internal static class JobRegistrationExtensions TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)); - public static IServiceCollection AddBuiltInConcelierJobs(this IServiceCollection services) + public static IServiceCollection AddBuiltInConcelierJobs(this IServiceCollection services, IConfiguration configuration) { ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + RegisterBuiltInJobDependencyInjectionRoutines(services, configuration); services.AddOptions() - .Configure((options, configuration) => + .Configure(options => { foreach (var registration in BaseBuiltInJobs) { @@ -220,4 +225,84 @@ internal static class JobRegistrationExtensions AddJobIfMissing(options, MergeReconcileBuiltInJob); } + + private static void RegisterBuiltInJobDependencyInjectionRoutines(IServiceCollection services, IConfiguration configuration) + { + var seenAssemblies = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var registration in BaseBuiltInJobs.Append(MergeReconcileBuiltInJob)) + { + var jobType = Type.GetType( + $"{registration.JobType}, {registration.AssemblyName}", + throwOnError: false, + ignoreCase: false); + + if (jobType is null) + { + continue; + } + + var assemblyKey = jobType.Assembly.FullName ?? jobType.Assembly.GetName().Name ?? jobType.Assembly.Location; + if (!seenAssemblies.Add(assemblyKey)) + { + continue; + } + + foreach (var routine in CreateRoutines(jobType.Assembly)) + { + var descriptorCountBefore = services.Count; + routine.Register(services, configuration); + RemoveValidateOnStartDescriptors(services, descriptorCountBefore); + } + } + } + + private static void RemoveValidateOnStartDescriptors(IServiceCollection services, int descriptorCountBefore) + { + for (var index = services.Count - 1; index >= descriptorCountBefore; index--) + { + var descriptor = services[index]; + if (descriptor.ServiceType == typeof(IStartupValidator) || IsStartupValidatorOptionsConfigurator(descriptor)) + { + services.RemoveAt(index); + } + } + } + + private static bool IsStartupValidatorOptionsConfigurator(ServiceDescriptor descriptor) + { + var serviceType = descriptor.ServiceType; + return serviceType.IsGenericType && + serviceType.GetGenericTypeDefinition() == typeof(IConfigureOptions<>) && + string.Equals( + serviceType.GenericTypeArguments[0].FullName, + "Microsoft.Extensions.Options.StartupValidatorOptions", + StringComparison.Ordinal); + } + + private static IEnumerable CreateRoutines(Assembly assembly) + { + Type[] types; + try + { + types = assembly.GetTypes(); + } + catch (ReflectionTypeLoadException ex) + { + types = ex.Types.Where(static type => type is not null).Select(static type => type!).ToArray(); + } + + foreach (var type in types) + { + if (type.IsAbstract || type.IsInterface || !typeof(IDependencyInjectionRoutine).IsAssignableFrom(type)) + { + continue; + } + + if (Activator.CreateInstance(type) is IDependencyInjectionRoutine routine) + { + yield return routine; + } + } + } } diff --git a/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs index e3dd92f1d..e7da4569e 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs @@ -4,6 +4,8 @@ using Microsoft.Extensions.Options; using StellaOps.Auth.ServerIntegration.Tenancy; using StellaOps.Concelier.Core.Jobs; using StellaOps.Concelier.Core.Sources; +using StellaOps.Concelier.WebService.Results; +using StellaOps.Concelier.WebService.Services; namespace StellaOps.Concelier.WebService.Extensions; @@ -15,6 +17,8 @@ internal static class SourceManagementEndpointExtensions { private const string AdvisoryReadPolicy = "Concelier.Advisories.Read"; private const string SourcesManagePolicy = "Concelier.Sources.Manage"; + private const string NotImplementedProblemType = "https://stellaops.org/problems/not-implemented"; + private const string NotImplementedErrorCode = "NOT_IMPLEMENTED"; public static void MapSourceManagementEndpoints(this WebApplication app) { @@ -43,22 +47,17 @@ internal static class SourceManagementEndpointExtensions // GET /status — enabled sources with last check results group.MapGet("/status", async ( - [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { - var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false); - var allSources = registry.GetAllSources(); - var items = new List(allSources.Count); - - foreach (var source in allSources) - { - items.Add(new SourceStatusItem + var items = (await configuredSources.GetStatusAsync(cancellationToken).ConfigureAwait(false)) + .Select(source => new SourceStatusItem { - SourceId = source.Id, - Enabled = enabledIds.Contains(source.Id, StringComparer.OrdinalIgnoreCase), - LastCheck = registry.GetLastCheckResult(source.Id) - }); - } + SourceId = source.SourceId, + Enabled = source.Enabled, + LastCheck = source.LastCheck + }) + .ToList(); return HttpResults.Ok(new SourceStatusResponse { Sources = items }); }) @@ -72,6 +71,7 @@ internal static class SourceManagementEndpointExtensions group.MapPost("/{sourceId}/enable", async ( string sourceId, [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { var source = registry.GetSource(sourceId); @@ -80,7 +80,7 @@ internal static class SourceManagementEndpointExtensions return HttpResults.NotFound(new { error = "source_not_found", sourceId }); } - var success = await registry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + var success = await configuredSources.EnableSourceAsync(sourceId, cancellationToken: cancellationToken).ConfigureAwait(false); return success ? HttpResults.Ok(new { sourceId, enabled = true }) : HttpResults.UnprocessableEntity(new { error = "enable_failed", sourceId }); @@ -97,6 +97,7 @@ internal static class SourceManagementEndpointExtensions group.MapPost("/{sourceId}/disable", async ( string sourceId, [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { var source = registry.GetSource(sourceId); @@ -105,7 +106,7 @@ internal static class SourceManagementEndpointExtensions return HttpResults.NotFound(new { error = "source_not_found", sourceId }); } - var success = await registry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + var success = await configuredSources.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); return success ? HttpResults.Ok(new { sourceId, enabled = false }) : HttpResults.UnprocessableEntity(new { error = "disable_failed", sourceId }); @@ -120,10 +121,10 @@ internal static class SourceManagementEndpointExtensions // POST /check — check all sources and auto-configure group.MapPost("/check", async ( - [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { - var result = await registry.CheckAllAndAutoConfigureAsync(cancellationToken).ConfigureAwait(false); + var result = await configuredSources.CheckAllAndPersistAsync(cancellationToken).ConfigureAwait(false); return HttpResults.Ok(result); }) .WithName("CheckAllSources") @@ -135,16 +136,15 @@ internal static class SourceManagementEndpointExtensions // POST /{sourceId}/check — check connectivity for a single source group.MapPost("/{sourceId}/check", async ( string sourceId, - [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { - var source = registry.GetSource(sourceId); - if (source is null) + var result = await configuredSources.CheckSourceConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false); + if (string.Equals(result.ErrorCode, "SOURCE_NOT_FOUND", StringComparison.OrdinalIgnoreCase)) { return HttpResults.NotFound(new { error = "source_not_found", sourceId }); } - var result = await registry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false); return HttpResults.Ok(result); }) .WithName("CheckSourceConnectivity") @@ -158,6 +158,7 @@ internal static class SourceManagementEndpointExtensions group.MapPost("/batch-enable", async ( [FromBody] BatchSourceRequest request, [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { if (request.SourceIds is null || request.SourceIds.Count == 0) @@ -175,7 +176,7 @@ internal static class SourceManagementEndpointExtensions continue; } - var success = await registry.EnableSourceAsync(id, cancellationToken).ConfigureAwait(false); + var success = await configuredSources.EnableSourceAsync(id, cancellationToken: cancellationToken).ConfigureAwait(false); results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "enable_failed" }); } @@ -192,6 +193,7 @@ internal static class SourceManagementEndpointExtensions group.MapPost("/batch-disable", async ( [FromBody] BatchSourceRequest request, [FromServices] ISourceRegistry registry, + [FromServices] ConfiguredAdvisorySourceService configuredSources, CancellationToken cancellationToken) => { if (request.SourceIds is null || request.SourceIds.Count == 0) @@ -209,7 +211,7 @@ internal static class SourceManagementEndpointExtensions continue; } - var success = await registry.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false); + var success = await configuredSources.DisableSourceAsync(id, cancellationToken).ConfigureAwait(false); results.Add(new BatchSourceResultItem { SourceId = id, Success = success, Error = success ? null : "disable_failed" }); } @@ -238,6 +240,11 @@ internal static class SourceManagementEndpointExtensions return HttpResults.NotFound(new { error = "source_not_found", sourceId }); } + if (coordinator is UnsupportedJobCoordinator) + { + return NotImplemented(httpContext, UnsupportedJobCoordinator.Detail); + } + // Backpressure: reject if active runs are at capacity var maxConcurrent = schedulerOptions.Value.MaxConcurrentJobs; var activeRuns = await coordinator.GetActiveRunsAsync(cancellationToken).ConfigureAwait(false); @@ -269,13 +276,19 @@ internal static class SourceManagementEndpointExtensions // POST /sync — trigger data sync for all enabled sources (batched with inter-batch delay) group.MapPost("/sync", async ( - [FromServices] ISourceRegistry registry, + HttpContext httpContext, + [FromServices] ConfiguredAdvisorySourceService configuredSources, [FromServices] IJobCoordinator coordinator, [FromServices] IOptions schedulerOptions, CancellationToken cancellationToken) => { + if (coordinator is UnsupportedJobCoordinator) + { + return NotImplemented(httpContext, UnsupportedJobCoordinator.Detail); + } + var opts = schedulerOptions.Value; - var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false); + var enabledIds = await configuredSources.GetEnabledSourceIdsAsync(cancellationToken).ConfigureAwait(false); var results = new List(enabledIds.Length); var batchSize = Math.Max(1, opts.MaxConcurrentJobs); var batchDelay = TimeSpan.FromSeconds(opts.SyncBatchDelaySeconds); @@ -370,6 +383,15 @@ internal static class SourceManagementEndpointExtensions runId = result.Run?.RunId }; } + + private static IResult NotImplemented(HttpContext context, string detail) => + ConcelierProblemResultFactory.Problem( + context, + NotImplementedProblemType, + "Operation not implemented", + StatusCodes.Status501NotImplemented, + NotImplementedErrorCode, + detail); } // ===== Response DTOs ===== diff --git a/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs b/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs index 334b3d6b9..9debdbfd7 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs @@ -210,6 +210,7 @@ public sealed class ConcelierOptions public bool Enabled { get; set; } public string ExportRoot { get; set; } = System.IO.Path.Combine("exports", "json"); + public string ImportRoot { get; set; } = System.IO.Path.Combine("imports", "mirror"); public string? ActiveExportId { get; set; } @@ -225,6 +226,9 @@ public sealed class ConcelierOptions [JsonIgnore] public string ExportRootAbsolute { get; internal set; } = string.Empty; + + [JsonIgnore] + public string ImportRootAbsolute { get; internal set; } = string.Empty; } public sealed class MirrorDomainOptions diff --git a/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsPostConfigure.cs b/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsPostConfigure.cs index 282afd736..985ae3388 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsPostConfigure.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsPostConfigure.cs @@ -53,6 +53,11 @@ public static class ConcelierOptionsPostConfigure mirror.ExportRoot = Path.Combine("exports", "json"); } + if (string.IsNullOrWhiteSpace(mirror.ImportRoot)) + { + mirror.ImportRoot = Path.Combine("imports", "mirror"); + } + var resolvedRoot = mirror.ExportRoot; if (!Path.IsPathRooted(resolvedRoot)) { @@ -61,6 +66,14 @@ public static class ConcelierOptionsPostConfigure mirror.ExportRootAbsolute = Path.GetFullPath(resolvedRoot); + var resolvedImportRoot = mirror.ImportRoot; + if (!Path.IsPathRooted(resolvedImportRoot)) + { + resolvedImportRoot = Path.Combine(contentRootPath, resolvedImportRoot); + } + + mirror.ImportRootAbsolute = Path.GetFullPath(resolvedImportRoot); + if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName)) { mirror.LatestDirectoryName = "latest"; diff --git a/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs b/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs index 91262eb9a..d99a17f77 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs @@ -225,11 +225,21 @@ public static class ConcelierOptionsValidator throw new InvalidOperationException("Mirror exportRoot must be configured."); } + if (string.IsNullOrWhiteSpace(mirror.ImportRoot)) + { + throw new InvalidOperationException("Mirror importRoot must be configured."); + } + if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute)) { throw new InvalidOperationException("Mirror export root could not be resolved."); } + if (string.IsNullOrWhiteSpace(mirror.ImportRootAbsolute)) + { + throw new InvalidOperationException("Mirror import root could not be resolved."); + } + if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName)) { throw new InvalidOperationException("Mirror latestDirectoryName must be provided."); diff --git a/src/Concelier/StellaOps.Concelier.WebService/Program.cs b/src/Concelier/StellaOps.Concelier.WebService/Program.cs index 1e10c0bbf..e9db682f0 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/Program.cs +++ b/src/Concelier/StellaOps.Concelier.WebService/Program.cs @@ -36,6 +36,7 @@ using StellaOps.Concelier.Core.Orchestration; using StellaOps.Concelier.Core.Raw; using StellaOps.Concelier.Core.Signals; using StellaOps.Concelier.Core.Sources; +using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; using StellaOps.Concelier.Merge; using StellaOps.Concelier.Merge.Services; using StellaOps.Concelier.Models; @@ -505,8 +506,7 @@ builder.Services.AddConcelierPostgresStorage(pgOptions => }); builder.Services.AddScoped(); -// Register in-memory lease store (single-instance dev mode). -builder.Services.AddSingleton(); +// Durable job lease coordination is registered by AddConcelierPostgresStorage. builder.Services.AddOptions() .Bind(builder.Configuration.GetSection("advisoryObservationEvents")) @@ -554,6 +554,13 @@ builder.Services.AddSingleton(); // Register signals services (CONCELIER-SIG-26-001) builder.Services.AddConcelierSignalsServices(); +if (!isTesting) +{ + builder.Services.RemoveAll(); + builder.Services.RemoveAll(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); +} // Register orchestration services (CONCELIER-ORCH-32-001) builder.Services.AddConcelierOrchestrationServices(); @@ -563,6 +570,7 @@ builder.Services.AddConcelierFederationServices(); // Register advisory source registry and connectivity services builder.Services.AddSourcesRegistry(builder.Configuration); +builder.Services.AddScoped(); // Mirror domain management is backed by Concelier PostgreSQL state so the // UI can operate against real backend persistence after fresh-volume resets. @@ -573,6 +581,9 @@ builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); +builder.Services.AddHttpContextAccessor(); builder.Services.AddSingleton(); // ── Topology Setup Services (in-memory stores, future: DB-backed) ── @@ -720,7 +731,8 @@ if (!features.NoMergeEnabled) } builder.Services.AddJobScheduler(); -builder.Services.AddBuiltInConcelierJobs(); +builder.Services.AddBuiltInConcelierJobs(builder.Configuration); +builder.Services.AddSingleton(); builder.Services.PostConfigure(options => { if (features.NoMergeEnabled) @@ -740,6 +752,27 @@ builder.Services.PostConfigure(options => } } }); + +if (!isTesting) +{ + builder.Services.RemoveAll(); + builder.Services.AddSingleton(); + builder.Services.RemoveAll(); + builder.Services.AddSingleton(); + builder.Services.RemoveAll(); + builder.Services.AddSingleton(); + + for (var index = builder.Services.Count - 1; index >= 0; index--) + { + var descriptor = builder.Services[index]; + if (descriptor.ServiceType == typeof(IHostedService) && + descriptor.ImplementationType == typeof(JobSchedulerHostedService)) + { + builder.Services.RemoveAt(index); + } + } +} + builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => new StellaOps.Concelier.WebService.Diagnostics.ServiceStatus(sp.GetRequiredService())); @@ -1108,6 +1141,7 @@ app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority); app.MapCanonicalAdvisoryEndpoints(); app.MapAdvisorySourceEndpoints(); app.MapSourceManagementEndpoints(); +app.MapInternalSetupSourceEndpoints(); app.MapInterestScoreEndpoints(); // Federation endpoints for site-to-site bundle sync @@ -1237,6 +1271,11 @@ orchestratorGroup.MapPost("/registry", async ( return tenantError; } + if (store is UnsupportedOrchestratorRegistryStore) + { + return RuntimeNotImplemented(context, UnsupportedOrchestratorRegistryStore.Detail); + } + if (string.IsNullOrWhiteSpace(request.ConnectorId) || string.IsNullOrWhiteSpace(request.Source)) { return Problem(context, "connectorId and source are required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId and source."); @@ -1278,6 +1317,11 @@ orchestratorGroup.MapPost("/heartbeat", async ( return tenantError; } + if (store is UnsupportedOrchestratorRegistryStore) + { + return RuntimeNotImplemented(context, UnsupportedOrchestratorRegistryStore.Detail); + } + if (string.IsNullOrWhiteSpace(request.ConnectorId)) { return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId."); @@ -1319,6 +1363,11 @@ orchestratorGroup.MapPost("/commands", async ( return tenantError; } + if (store is UnsupportedOrchestratorRegistryStore) + { + return RuntimeNotImplemented(context, UnsupportedOrchestratorRegistryStore.Detail); + } + if (string.IsNullOrWhiteSpace(request.ConnectorId)) { return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId."); @@ -1367,6 +1416,11 @@ orchestratorGroup.MapGet("/commands", async ( return tenantError; } + if (store is UnsupportedOrchestratorRegistryStore) + { + return RuntimeNotImplemented(context, UnsupportedOrchestratorRegistryStore.Detail); + } + if (string.IsNullOrWhiteSpace(connectorId)) { return Problem(context, "connectorId is required", StatusCodes.Status400BadRequest, ProblemTypes.Validation, "Provide connectorId."); @@ -3491,6 +3545,24 @@ IResult Problem(HttpContext context, string title, int statusCode, string type, return HttpResults.Content(payload, "application/problem+json", Encoding.UTF8, statusCode); } +IResult RuntimeNotImplemented(HttpContext context, string detail) +{ + return ConcelierProblemResultFactory.Problem( + context, + "https://stellaops.org/problems/not-implemented", + "Operation not implemented", + StatusCodes.Status501NotImplemented, + "NOT_IMPLEMENTED", + detail); +} + +IResult? EnsureAffectedSymbolsRuntimeSupported(HttpContext context, IAffectedSymbolProvider symbolProvider) +{ + return symbolProvider is UnsupportedAffectedSymbolProvider + ? RuntimeNotImplemented(context, UnsupportedAffectedSymbolProvider.Detail) + : null; +} + bool TryResolveTenant(HttpContext context, bool requireHeader, out string tenant, out IResult? error) { tenant = string.Empty; @@ -4117,6 +4189,11 @@ var jobsListEndpoint = app.MapGet("/jobs", async (string? kind, int? limit, [Fro { ApplyNoCache(context.Response); + if (coordinator is UnsupportedJobCoordinator) + { + return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); + } + var take = Math.Clamp(limit.GetValueOrDefault(50), 1, 200); var runs = await coordinator.GetRecentRunsAsync(kind, take, cancellationToken).ConfigureAwait(false); var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); @@ -4131,6 +4208,11 @@ var jobByIdEndpoint = app.MapGet("/jobs/{runId:guid}", async (Guid runId, [FromS { ApplyNoCache(context.Response); + if (coordinator is UnsupportedJobCoordinator) + { + return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); + } + var run = await coordinator.GetRunAsync(runId, cancellationToken).ConfigureAwait(false); if (run is null) { @@ -4148,6 +4230,11 @@ var jobDefinitionsEndpoint = app.MapGet("/jobs/definitions", async ([FromService { ApplyNoCache(context.Response); + if (coordinator is UnsupportedJobCoordinator) + { + return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); + } + var definitions = await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false); if (definitions.Count == 0) { @@ -4175,6 +4262,11 @@ var jobDefinitionEndpoint = app.MapGet("/jobs/definitions/{kind}", async (string { ApplyNoCache(context.Response); + if (coordinator is UnsupportedJobCoordinator) + { + return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); + } + var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false)) .FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal)); @@ -4198,6 +4290,11 @@ var jobDefinitionRunsEndpoint = app.MapGet("/jobs/definitions/{kind}/runs", asyn { ApplyNoCache(context.Response); + if (coordinator is UnsupportedJobCoordinator) + { + return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); + } + var definition = (await coordinator.GetDefinitionsAsync(cancellationToken).ConfigureAwait(false)) .FirstOrDefault(d => string.Equals(d.Kind, kind, StringComparison.Ordinal)); @@ -4220,6 +4317,11 @@ var activeJobsEndpoint = app.MapGet("/jobs/active", async ([FromServices] IJobCo { ApplyNoCache(context.Response); + if (coordinator is UnsupportedJobCoordinator) + { + return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); + } + var runs = await coordinator.GetActiveRunsAsync(cancellationToken).ConfigureAwait(false); var payload = runs.Select(JobRunResponse.FromSnapshot).ToArray(); return JsonResult(payload); @@ -4233,6 +4335,11 @@ var triggerJobEndpoint = app.MapPost("/jobs/{*jobKind}", async (string jobKind, { ApplyNoCache(context.Response); + if (coordinator is UnsupportedJobCoordinator) + { + return RuntimeNotImplemented(context, UnsupportedJobCoordinator.Detail); + } + request ??= new JobTriggerRequest(); request.Parameters ??= new Dictionary(StringComparer.Ordinal); var trigger = string.IsNullOrWhiteSpace(request.Trigger) ? "api" : request.Trigger; @@ -4433,6 +4540,12 @@ app.MapGet("/v1/signals/symbols", async ( return authorizationError; } + var runtimeError = EnsureAffectedSymbolsRuntimeSupported(context, symbolProvider); + if (runtimeError is not null) + { + return runtimeError; + } + // Parse symbol types if provided ImmutableArray? symbolTypes = null; if (!string.IsNullOrWhiteSpace(symbolType)) @@ -4503,6 +4616,12 @@ app.MapGet("/v1/signals/symbols/advisory/{advisoryId}", async ( return ConcelierProblemResultFactory.AdvisoryIdRequired(context); } + var runtimeError = EnsureAffectedSymbolsRuntimeSupported(context, symbolProvider); + if (runtimeError is not null) + { + return runtimeError; + } + var symbolSet = await symbolProvider.GetByAdvisoryAsync(tenant!, advisoryId.Trim(), cancellationToken); return HttpResults.Ok(ToSymbolSetResponse(symbolSet)); @@ -4536,6 +4655,12 @@ app.MapGet("/v1/signals/symbols/package/{*purl}", async ( type: "https://stellaops.org/problems/validation"); } + var runtimeError = EnsureAffectedSymbolsRuntimeSupported(context, symbolProvider); + if (runtimeError is not null) + { + return runtimeError; + } + var symbolSet = await symbolProvider.GetByPackageAsync(tenant!, purl.Trim(), cancellationToken); return HttpResults.Ok(ToSymbolSetResponse(symbolSet)); @@ -4578,6 +4703,12 @@ app.MapPost("/v1/signals/symbols/batch", async ( type: "https://stellaops.org/problems/validation"); } + var runtimeError = EnsureAffectedSymbolsRuntimeSupported(context, symbolProvider); + if (runtimeError is not null) + { + return runtimeError; + } + var results = await symbolProvider.GetByAdvisoriesBatchAsync(tenant!, request.AdvisoryIds, cancellationToken); var response = new SignalsSymbolBatchResponse( @@ -4612,6 +4743,12 @@ app.MapGet("/v1/signals/symbols/exists/{advisoryId}", async ( return ConcelierProblemResultFactory.AdvisoryIdRequired(context); } + var runtimeError = EnsureAffectedSymbolsRuntimeSupported(context, symbolProvider); + if (runtimeError is not null) + { + return runtimeError; + } + var exists = await symbolProvider.HasSymbolsAsync(tenant!, advisoryId.Trim(), cancellationToken); return HttpResults.Ok(new SignalsSymbolExistsResponse(Exists: exists, AdvisoryId: advisoryId.Trim())); diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/ConfiguredAdvisorySourceService.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/ConfiguredAdvisorySourceService.cs new file mode 100644 index 000000000..4606c995e --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/ConfiguredAdvisorySourceService.cs @@ -0,0 +1,813 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Net; +using System.Net.Http; +using System.Security.Authentication; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Core.Sources; +using StellaOps.Concelier.Persistence.Postgres.Models; +using StellaOps.Concelier.Persistence.Postgres.Repositories; +using StellaOps.Concelier.WebService.Extensions; + +namespace StellaOps.Concelier.WebService.Services; + +/// +/// Persists advisory/VEX source configuration so setup, management UI, and restarts share one truth model. +/// +public sealed class ConfiguredAdvisorySourceService +{ + private const string MirrorMode = "mirror"; + private const string ManualMode = "custom"; + private const string MirrorSourceId = "stella-mirror"; + private const string MirrorConsumerHttpClientName = "MirrorConsumer"; + private const string MirrorIndexPath = "/concelier/exports/index.json"; + private static readonly TimeSpan SetupConnectivityTimeout = TimeSpan.FromSeconds(10); + + private readonly ISourceRepository sourceRepository; + private readonly ISourceRegistry sourceRegistry; + private readonly IHttpClientFactory httpClientFactory; + private readonly IMirrorConfigStore mirrorConfigStore; + private readonly IMirrorConsumerConfigStore mirrorConsumerConfigStore; + private readonly TimeProvider timeProvider; + private readonly ILogger logger; + + public ConfiguredAdvisorySourceService( + ISourceRepository sourceRepository, + ISourceRegistry sourceRegistry, + IHttpClientFactory httpClientFactory, + IMirrorConfigStore mirrorConfigStore, + IMirrorConsumerConfigStore mirrorConsumerConfigStore, + TimeProvider timeProvider, + ILogger logger) + { + this.sourceRepository = sourceRepository ?? throw new ArgumentNullException(nameof(sourceRepository)); + this.sourceRegistry = sourceRegistry ?? throw new ArgumentNullException(nameof(sourceRegistry)); + this.httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + this.mirrorConfigStore = mirrorConfigStore ?? throw new ArgumentNullException(nameof(mirrorConfigStore)); + this.mirrorConsumerConfigStore = mirrorConsumerConfigStore ?? throw new ArgumentNullException(nameof(mirrorConsumerConfigStore)); + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> GetStatusAsync(CancellationToken cancellationToken = default) + { + var persisted = await sourceRepository.ListAsync(enabled: null, cancellationToken).ConfigureAwait(false); + var persistedByKey = persisted.ToDictionary(source => source.Key, StringComparer.OrdinalIgnoreCase); + + return sourceRegistry.GetAllSources() + .Select(source => new ConfiguredAdvisorySourceStatus( + source.Id, + persistedByKey.TryGetValue(source.Id, out var entity) && entity.Enabled, + sourceRegistry.GetLastCheckResult(source.Id))) + .ToImmutableArray(); + } + + public async Task> GetEnabledSourceIdsAsync(CancellationToken cancellationToken = default) + { + var persisted = await sourceRepository.ListAsync(enabled: true, cancellationToken).ConfigureAwait(false); + return persisted + .Select(source => source.Key) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(key => key, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + public async Task EnableSourceAsync( + string sourceId, + IReadOnlyDictionary? configValues = null, + CancellationToken cancellationToken = default) + { + var definition = sourceRegistry.GetSource(sourceId); + if (definition is null) + { + return false; + } + + var existing = await sourceRepository.GetByKeyAsync(sourceId, cancellationToken).ConfigureAwait(false); + var mirrorUrl = string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) + ? NormalizeMirrorUrl(GetConfigValue(configValues, "sources.mirror.url") ?? existing?.Url ?? definition.BaseEndpoint) + : null; + await UpsertSourceAsync( + definition, + enabled: true, + existing, + configValues, + string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? MirrorMode : ManualMode, + mirrorUrl, + configuredBy: "source-management", + cancellationToken).ConfigureAwait(false); + + if (string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + { + await ApplyMirrorConsumerConfigurationAsync(mirrorUrl!, cancellationToken).ConfigureAwait(false); + } + + await sourceRegistry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + return true; + } + + public async Task DisableSourceAsync(string sourceId, CancellationToken cancellationToken = default) + { + var definition = sourceRegistry.GetSource(sourceId); + if (definition is null) + { + return false; + } + + var existing = await sourceRepository.GetByKeyAsync(sourceId, cancellationToken).ConfigureAwait(false); + await UpsertSourceAsync( + definition, + enabled: false, + existing, + configValues: null, + mode: string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? MirrorMode : ManualMode, + mirrorUrl: existing?.Url, + configuredBy: "source-management", + cancellationToken).ConfigureAwait(false); + + if (string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + { + await SetMirrorModeAsync(enabled: false, consumerBaseAddress: null, cancellationToken).ConfigureAwait(false); + } + + await sourceRegistry.DisableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + return true; + } + + public async Task CheckSourceConnectivityAsync( + string sourceId, + CancellationToken cancellationToken = default) + { + var definition = sourceRegistry.GetSource(sourceId); + if (definition is null) + { + return SourceConnectivityResult.NotFound(sourceId); + } + + if (!string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + { + return await sourceRegistry.CheckConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false); + } + + var existing = await sourceRepository.GetByKeyAsync(sourceId, cancellationToken).ConfigureAwait(false); + var mirrorUrl = NormalizeMirrorUrl(existing?.Url ?? definition.BaseEndpoint); + return await ProbeMirrorConnectivityAsync(mirrorUrl, cancellationToken).ConfigureAwait(false); + } + + public async Task CheckAllAndPersistAsync(CancellationToken cancellationToken = default) + { + var checkedAt = timeProvider.GetUtcNow(); + var started = Stopwatch.GetTimestamp(); + var results = ImmutableArray.CreateBuilder(); + + foreach (var definition in sourceRegistry.GetAllSources().OrderBy(source => source.Id, StringComparer.OrdinalIgnoreCase)) + { + var result = await CheckSourceConnectivityAsync(definition.Id, cancellationToken).ConfigureAwait(false); + results.Add(result); + var existing = await sourceRepository.GetByKeyAsync(definition.Id, cancellationToken).ConfigureAwait(false); + await UpsertSourceAsync( + definition, + enabled: result.IsHealthy, + existing, + configValues: null, + mode: string.Equals(definition.Id, MirrorSourceId, StringComparison.OrdinalIgnoreCase) ? MirrorMode : ManualMode, + mirrorUrl: existing?.Url, + configuredBy: "source-management", + cancellationToken).ConfigureAwait(false); + } + + return SourceCheckResult.FromResults( + results, + checkedAt, + Stopwatch.GetElapsedTime(started)); + } + + public async Task ProbeSetupAsync( + IReadOnlyDictionary? configValues, + CancellationToken cancellationToken = default) + { + var instruction = ParseInstruction(configValues); + var connectivity = await ProbeInstructionAsync(instruction, cancellationToken).ConfigureAwait(false); + + return new SetupAdvisorySourcesBootstrapResponse( + instruction.Mode, + instruction.EnabledSourceIds, + instruction.Message, + instruction.MirrorUrl, + connectivity.Ready, + connectivity.Message, + Applied: false); + } + + public async Task ApplySetupAsync( + IReadOnlyDictionary? configValues, + CancellationToken cancellationToken = default) + { + var instruction = ParseInstruction(configValues); + var connectivity = await ProbeInstructionAsync(instruction, cancellationToken).ConfigureAwait(false); + if (!connectivity.Ready) + { + logger.LogWarning( + "Rejected advisory/VEX bootstrap apply in {Mode} mode because connectivity is not ready: {Message}", + instruction.Mode, + connectivity.Message); + + return new SetupAdvisorySourcesBootstrapResponse( + instruction.Mode, + instruction.EnabledSourceIds, + $"Advisory/VEX configuration was not applied. {connectivity.Message}", + instruction.MirrorUrl, + connectivity.Ready, + connectivity.Message, + Applied: false); + } + + var persisted = await sourceRepository.ListAsync(enabled: null, cancellationToken).ConfigureAwait(false); + var selected = instruction.EnabledSourceIds.ToHashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var sourceId in instruction.EnabledSourceIds) + { + var definition = sourceRegistry.GetSource(sourceId) + ?? throw new InvalidOperationException($"Source '{sourceId}' is not registered."); + var existing = persisted.FirstOrDefault(source => string.Equals(source.Key, sourceId, StringComparison.OrdinalIgnoreCase)); + + await UpsertSourceAsync( + definition, + enabled: true, + existing, + configValues, + instruction.Mode, + instruction.MirrorUrl, + "setup-wizard", + cancellationToken).ConfigureAwait(false); + + if (string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + { + await ApplyMirrorConsumerConfigurationAsync(instruction.MirrorUrl!, cancellationToken).ConfigureAwait(false); + } + + await sourceRegistry.EnableSourceAsync(sourceId, cancellationToken).ConfigureAwait(false); + } + + foreach (var existing in persisted) + { + if (selected.Contains(existing.Key)) + { + continue; + } + + var definition = sourceRegistry.GetSource(existing.Key); + if (definition is null) + { + continue; + } + + await UpsertSourceAsync( + definition, + enabled: false, + existing, + configValues: null, + mode: instruction.Mode, + mirrorUrl: existing.Url, + configuredBy: "setup-wizard", + cancellationToken).ConfigureAwait(false); + + await sourceRegistry.DisableSourceAsync(existing.Key, cancellationToken).ConfigureAwait(false); + } + + logger.LogInformation( + "Applied advisory/VEX bootstrap configuration in {Mode} mode for sources: {Sources}", + instruction.Mode, + string.Join(", ", instruction.EnabledSourceIds)); + + return new SetupAdvisorySourcesBootstrapResponse( + instruction.Mode, + instruction.EnabledSourceIds, + instruction.Message, + instruction.MirrorUrl, + connectivity.Ready, + connectivity.Message, + Applied: true); + } + + private SetupAdvisorySourceInstruction ParseInstruction(IReadOnlyDictionary? configValues) + { + var mode = NormalizeMode(GetConfigValue(configValues, "sources.mode")); + if (string.Equals(mode, MirrorMode, StringComparison.OrdinalIgnoreCase)) + { + var mirrorUrl = NormalizeMirrorUrl(GetConfigValue(configValues, "sources.mirror.url") ?? SourceDefinitions.StellaMirror.BaseEndpoint); + if (!Uri.TryCreate(mirrorUrl, UriKind.Absolute, out _)) + { + throw new InvalidOperationException("StellaOps Mirror URL must be an absolute URI."); + } + + return new SetupAdvisorySourceInstruction( + MirrorMode, + [MirrorSourceId], + $"StellaOps Mirror will be enabled at {mirrorUrl}. Initial aggregation starts automatically after apply.", + mirrorUrl); + } + + if (!string.Equals(mode, ManualMode, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Advisory source mode must be either 'mirror' or 'custom'."); + } + + var selected = ParseSelectedSourceIds(configValues); + if (selected.Length == 0) + { + throw new InvalidOperationException("Select at least one advisory or VEX source, or skip this step and configure sources later."); + } + + var unknown = selected + .Where(sourceId => sourceRegistry.GetSource(sourceId) is null) + .OrderBy(sourceId => sourceId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + if (unknown.Length > 0) + { + throw new InvalidOperationException($"Unknown advisory/VEX sources: {string.Join(", ", unknown)}."); + } + + if (selected.Any(sourceId => string.Equals(sourceId, MirrorSourceId, StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidOperationException("StellaOps Mirror is configured through mirror mode. Switch modes instead of selecting it manually."); + } + + return new SetupAdvisorySourceInstruction( + ManualMode, + selected, + $"Manual advisory/VEX mode will enable {selected.Length} source(s): {string.Join(", ", selected)}.", + MirrorUrl: null); + } + + private async Task ProbeInstructionAsync( + SetupAdvisorySourceInstruction instruction, + CancellationToken cancellationToken) + { + if (string.Equals(instruction.Mode, MirrorMode, StringComparison.OrdinalIgnoreCase)) + { + var result = await ProbeMirrorConnectivityAsync(instruction.MirrorUrl!, cancellationToken).ConfigureAwait(false); + var successMessage = $"Verified StellaOps Mirror export index at {ComposeMirrorProbeUrl(instruction.MirrorUrl!)}."; + return new SetupAdvisorySourceConnectivitySummary( + result.IsHealthy, + result.IsHealthy + ? successMessage + : result.ErrorMessage ?? $"StellaOps Mirror at {ComposeMirrorProbeUrl(instruction.MirrorUrl!)} is not reachable."); + } + + var failures = new List(); + foreach (var sourceId in instruction.EnabledSourceIds) + { + var result = await CheckSourceConnectivityAsync(sourceId, cancellationToken).ConfigureAwait(false); + if (!result.IsHealthy) + { + failures.Add($"{sourceId}: {result.ErrorMessage ?? result.Status.ToString()}"); + } + } + + if (failures.Count == 0) + { + return new SetupAdvisorySourceConnectivitySummary( + Ready: true, + Message: $"Verified connectivity for {instruction.EnabledSourceIds.Length} selected advisory/VEX source(s)."); + } + + return new SetupAdvisorySourceConnectivitySummary( + Ready: false, + Message: $"Connectivity failed for {failures.Count} selected source(s). {string.Join(" ", failures)}"); + } + + private async Task UpsertSourceAsync( + SourceDefinition definition, + bool enabled, + SourceEntity? existing, + IReadOnlyDictionary? configValues, + string mode, + string? mirrorUrl, + string configuredBy, + CancellationToken cancellationToken) + { + var now = timeProvider.GetUtcNow(); + var config = BuildSourceConfig(definition, configValues, mode, mirrorUrl, existing?.Config); + var metadata = BuildSourceMetadata(existing?.Metadata, configuredBy, mode); + + var entity = new SourceEntity + { + Id = existing?.Id ?? Guid.NewGuid(), + Key = definition.Id, + Name = definition.DisplayName, + SourceType = definition.Id, + Url = string.Equals(definition.Id, MirrorSourceId, StringComparison.OrdinalIgnoreCase) + ? mirrorUrl ?? existing?.Url ?? definition.BaseEndpoint + : existing?.Url ?? definition.BaseEndpoint, + Priority = existing?.Priority ?? definition.DefaultPriority, + Enabled = enabled, + Config = config, + Metadata = metadata, + CreatedAt = existing?.CreatedAt ?? now, + UpdatedAt = now + }; + + await sourceRepository.UpsertAsync(entity, cancellationToken).ConfigureAwait(false); + } + + private static string BuildSourceConfig( + SourceDefinition definition, + IReadOnlyDictionary? configValues, + string mode, + string? mirrorUrl, + string? existingJson) + { + var builder = ParseJsonObject(existingJson); + + if (string.Equals(definition.Id, MirrorSourceId, StringComparison.OrdinalIgnoreCase)) + { + builder["mode"] = mode; + builder["baseUrl"] = mirrorUrl ?? SourceDefinitions.StellaMirror.BaseEndpoint; + + var apiKey = GetConfigValue(configValues, "sources.mirror.apiKey"); + if (!string.IsNullOrWhiteSpace(apiKey)) + { + builder["apiKey"] = apiKey; + } + + return JsonSerializer.Serialize(builder); + } + + var prefix = $"sources.{definition.Id}."; + if (configValues is not null) + { + foreach (var pair in configValues) + { + if (!pair.Key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var field = pair.Key[prefix.Length..]; + if (string.Equals(field, "enabled", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + builder[field] = pair.Value; + } + } + + builder["mode"] = mode; + return JsonSerializer.Serialize(builder); + } + + private static string BuildSourceMetadata(string? existingJson, string configuredBy, string mode) + { + var builder = ParseJsonObject(existingJson); + builder["configuredBy"] = configuredBy; + builder["mode"] = mode; + return JsonSerializer.Serialize(builder); + } + + private static Dictionary ParseJsonObject(string? json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + try + { + using var document = JsonDocument.Parse(json); + if (document.RootElement.ValueKind != JsonValueKind.Object) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var property in document.RootElement.EnumerateObject()) + { + result[property.Name] = property.Value.ValueKind switch + { + JsonValueKind.String => property.Value.GetString(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Number => property.Value.TryGetInt64(out var integer) + ? integer + : property.Value.GetDouble(), + _ => property.Value.GetRawText() + }; + } + + return result; + } + catch + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + } + + private static ImmutableArray ParseSelectedSourceIds(IReadOnlyDictionary? configValues) + { + var selected = new HashSet(StringComparer.OrdinalIgnoreCase); + if (configValues is null) + { + return ImmutableArray.Empty; + } + + if (configValues.TryGetValue("sources.enabled", out var enabledList) && !string.IsNullOrWhiteSpace(enabledList)) + { + foreach (var sourceId in enabledList.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + selected.Add(sourceId); + } + } + + foreach (var pair in configValues) + { + if (!pair.Key.StartsWith("sources.", StringComparison.OrdinalIgnoreCase) || + !pair.Key.EndsWith(".enabled", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var sourceId = pair.Key["sources.".Length..^".enabled".Length]; + if (string.IsNullOrWhiteSpace(sourceId)) + { + continue; + } + + if (bool.TryParse(pair.Value, out var enabled)) + { + if (enabled) + { + selected.Add(sourceId); + } + else + { + selected.Remove(sourceId); + } + } + } + + return selected + .OrderBy(sourceId => sourceId, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static string NormalizeMode(string? mode) => + string.Equals(mode, ManualMode, StringComparison.OrdinalIgnoreCase) + ? ManualMode + : MirrorMode; + + private static string? GetConfigValue(IReadOnlyDictionary? configValues, string key) + { + if (configValues is null) + { + return null; + } + + return configValues.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value) + ? value.Trim() + : null; + } + + private async Task ProbeMirrorConnectivityAsync( + string mirrorUrl, + CancellationToken cancellationToken) + { + var normalizedMirrorUrl = NormalizeMirrorUrl(mirrorUrl); + var probeUrl = ComposeMirrorProbeUrl(normalizedMirrorUrl); + var checkedAt = timeProvider.GetUtcNow(); + var stopwatch = Stopwatch.StartNew(); + + try + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(SetupConnectivityTimeout); + + var client = httpClientFactory.CreateClient(MirrorConsumerHttpClientName); + using var response = await client.GetAsync( + probeUrl, + HttpCompletionOption.ResponseHeadersRead, + timeout.Token).ConfigureAwait(false); + + stopwatch.Stop(); + + if (response.IsSuccessStatusCode) + { + return SourceConnectivityResult.Healthy(MirrorSourceId, stopwatch.Elapsed, checkedAt) with + { + Diagnostics = ImmutableDictionary.Empty.Add("probeUrl", probeUrl) + }; + } + + var message = response.StatusCode == HttpStatusCode.NotFound + ? $"StellaOps Mirror did not expose {MirrorIndexPath} at {probeUrl}." + : $"StellaOps Mirror returned HTTP {(int)response.StatusCode} from {probeUrl}."; + + return SourceConnectivityResult.Failed( + MirrorSourceId, + $"MIRROR_HTTP_{(int)response.StatusCode}", + message, + response.StatusCode == HttpStatusCode.NotFound + ? ImmutableArray.Create( + "The configured mirror base URL is reachable, but the advisory export index is missing.") + : ImmutableArray.Create( + "The configured mirror endpoint responded, but it did not return a success status."), + response.StatusCode == HttpStatusCode.NotFound + ? ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = $"Verify the upstream mirror publishes {MirrorIndexPath}." + }, + new RemediationStep + { + Order = 2, + Description = "If this is a local or reverse-proxied mirror, confirm the routed hostname and path match the configured mirror URL." + }) + : ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = "Verify the mirror URL, authentication requirements, and reverse-proxy exposure." + }), + checkedAt, + stopwatch.Elapsed, + (int)response.StatusCode) with + { + Diagnostics = ImmutableDictionary.Empty.Add("probeUrl", probeUrl) + }; + } + catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + stopwatch.Stop(); + var isTlsFailure = LooksLikeTlsFailure(ex); + var message = BuildMirrorExceptionMessage(probeUrl, ex, isTlsFailure); + var reasons = isTlsFailure + ? ImmutableArray.Create( + "The certificate presented by the mirror does not match the configured hostname, or the certificate chain is not trusted by the runtime.", + "Local browser automation can ignore dev certificates, but product advisory aggregation requires a hostname-valid certificate.") + : ImmutableArray.Create( + "The configured mirror URL may be unreachable on the network.", + "The upstream mirror may be down or incorrectly reverse-proxied."); + var remediation = isTlsFailure + ? ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = "Use a mirror hostname covered by the served certificate, or issue a certificate whose SAN/CN matches the configured mirror URL." + }, + new RemediationStep + { + Order = 2, + Description = "Keep local certificate exemptions in automation only; product advisory sync will continue to enforce TLS hostname validation." + }) + : ImmutableArray.Create( + new RemediationStep + { + Order = 1, + Description = "Verify the configured mirror URL is reachable from the Concelier runtime." + }); + + return SourceConnectivityResult.Failed( + MirrorSourceId, + isTlsFailure ? "MIRROR_TLS_FAILURE" : "MIRROR_UNREACHABLE", + message, + reasons, + remediation, + checkedAt, + stopwatch.Elapsed) with + { + Diagnostics = ImmutableDictionary.Empty + .Add("probeUrl", probeUrl) + .Add("exceptionType", ex.GetType().FullName ?? ex.GetType().Name) + }; + } + } + + private async Task ApplyMirrorConsumerConfigurationAsync(string mirrorUrl, CancellationToken cancellationToken) + { + var consumerBaseAddress = NormalizeMirrorUrl(mirrorUrl); + await SetMirrorModeAsync(enabled: true, consumerBaseAddress, cancellationToken).ConfigureAwait(false); + await mirrorConsumerConfigStore.SetConsumerConfigAsync(new ConsumerConfigRequest + { + BaseAddress = consumerBaseAddress, + DomainId = "primary", + IndexPath = "/concelier/exports/index.json", + }, cancellationToken).ConfigureAwait(false); + } + + private Task SetMirrorModeAsync(bool enabled, string? consumerBaseAddress, CancellationToken cancellationToken) + { + return mirrorConfigStore.UpdateConfigAsync(new UpdateMirrorConfigRequest + { + Mode = enabled ? "Mirror" : "Direct", + ConsumerBaseAddress = consumerBaseAddress ?? string.Empty, + }, cancellationToken); + } + + private static string NormalizeMirrorUrl(string mirrorUrl) + { + if (!Uri.TryCreate(mirrorUrl, UriKind.Absolute, out var uri)) + { + return mirrorUrl; + } + + var path = uri.AbsolutePath.TrimEnd('/'); + if (path.EndsWith("/api/v1", StringComparison.OrdinalIgnoreCase)) + { + path = path[..^"/api/v1".Length]; + } + + if (string.IsNullOrWhiteSpace(path)) + { + path = "/"; + } + + var builder = new UriBuilder(uri) + { + Path = path, + Query = string.Empty, + Fragment = string.Empty, + }; + + return builder.Uri.ToString().TrimEnd('/'); + } + + private static string ComposeMirrorProbeUrl(string mirrorUrl) + => $"{NormalizeMirrorUrl(mirrorUrl).TrimEnd('/')}{MirrorIndexPath}"; + + private static string BuildMirrorExceptionMessage(string probeUrl, Exception exception, bool isTlsFailure) + { + var message = $"StellaOps Mirror at {probeUrl} could not be reached: {GetInnermostMessage(exception)}."; + if (!isTlsFailure) + { + return message; + } + + return $"{message} Product advisory sync requires a hostname-valid certificate. Local automation may ignore local certificates, but runtime aggregation will not."; + } + + private static bool LooksLikeTlsFailure(Exception exception) + { + for (Exception? current = exception; current is not null; current = current.InnerException) + { + if (current is AuthenticationException) + { + return true; + } + + if (current.Message.Contains("certificate", StringComparison.OrdinalIgnoreCase) || + current.Message.Contains("ssl", StringComparison.OrdinalIgnoreCase) || + current.Message.Contains("tls", StringComparison.OrdinalIgnoreCase) || + current.Message.Contains("RemoteCertificateNameMismatch", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static string GetInnermostMessage(Exception exception) + { + var current = exception; + while (current.InnerException is not null) + { + current = current.InnerException; + } + + return current.Message; + } + + private sealed record SetupAdvisorySourceInstruction( + string Mode, + ImmutableArray EnabledSourceIds, + string Message, + string? MirrorUrl); + + private sealed record SetupAdvisorySourceConnectivitySummary( + bool Ready, + string Message); +} + +public sealed record ConfiguredAdvisorySourceStatus( + string SourceId, + bool Enabled, + SourceConnectivityResult? LastCheck); + +public sealed record SetupAdvisorySourcesBootstrapRequest( + IReadOnlyDictionary? ConfigValues); + +public sealed record SetupAdvisorySourcesBootstrapResponse( + string Mode, + IReadOnlyList EnabledSourceIds, + string Message, + string? MirrorUrl, + bool ConnectivityReady, + string ConnectivityMessage, + bool Applied); diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/ImmediateSourceSyncTrigger.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/ImmediateSourceSyncTrigger.cs new file mode 100644 index 000000000..7ec8e43c2 --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/ImmediateSourceSyncTrigger.cs @@ -0,0 +1,164 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Core.Sources; +using System.Collections.Concurrent; +using System.Collections.Immutable; + +namespace StellaOps.Concelier.WebService.Services; + +/// +/// Live-safe trigger used by source enablement flows to start a source pipeline without enabling the deprecated jobs runtime. +/// +public sealed class ImmediateSourceSyncTrigger : ISourceSyncTrigger +{ + private static readonly IReadOnlyList PipelineStages = ["fetch", "parse", "map"]; + private static readonly IReadOnlyDictionary EmptyParameters = new Dictionary(StringComparer.Ordinal); + + private readonly IServiceScopeFactory serviceScopeFactory; + private readonly ILogger logger; + private readonly ILoggerFactory loggerFactory; + private readonly TimeProvider timeProvider; + private readonly IHostApplicationLifetime applicationLifetime; + private readonly JobSchedulerOptions schedulerOptions; + private readonly SemaphoreSlim concurrencyGate; + private readonly ConcurrentDictionary activeRuns = new(StringComparer.OrdinalIgnoreCase); + + public ImmediateSourceSyncTrigger( + IServiceScopeFactory serviceScopeFactory, + ILogger logger, + ILoggerFactory loggerFactory, + TimeProvider timeProvider, + IHostApplicationLifetime applicationLifetime, + IOptions schedulerOptions) + { + this.serviceScopeFactory = serviceScopeFactory ?? throw new ArgumentNullException(nameof(serviceScopeFactory)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + this.applicationLifetime = applicationLifetime ?? throw new ArgumentNullException(nameof(applicationLifetime)); + this.schedulerOptions = (schedulerOptions ?? throw new ArgumentNullException(nameof(schedulerOptions))).Value; + concurrencyGate = new SemaphoreSlim(Math.Max(1, this.schedulerOptions.MaxConcurrentJobs), Math.Max(1, this.schedulerOptions.MaxConcurrentJobs)); + } + + public Task TriggerAsync( + string sourceId, + string trigger, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceId); + ArgumentException.ThrowIfNullOrWhiteSpace(trigger); + + var pipeline = ResolvePipeline(sourceId); + if (pipeline.Length == 0) + { + return Task.FromResult(JobTriggerResult.NotFound($"No live sync pipeline is registered for source '{sourceId}'.")); + } + + if (activeRuns.TryGetValue(sourceId, out _)) + { + return Task.FromResult(JobTriggerResult.AlreadyRunning($"Source '{sourceId}' is already syncing.")); + } + + var now = timeProvider.GetUtcNow(); + var run = new JobRunSnapshot( + RunId: Guid.NewGuid(), + Kind: pipeline[0].Kind, + Status: JobRunStatus.Pending, + CreatedAt: now, + StartedAt: now, + CompletedAt: null, + Trigger: trigger, + ParametersHash: null, + Error: null, + Timeout: null, + LeaseDuration: null, + Parameters: EmptyParameters); + + if (!activeRuns.TryAdd(sourceId, run)) + { + return Task.FromResult(JobTriggerResult.AlreadyRunning($"Source '{sourceId}' is already syncing.")); + } + + _ = Task.Run(() => ExecutePipelineAsync(sourceId, trigger, run, pipeline), CancellationToken.None); + return Task.FromResult(JobTriggerResult.Accepted(run)); + } + + private ImmutableArray ResolvePipeline(string sourceId) + { + var definitions = ImmutableArray.CreateBuilder(); + + foreach (var stage in PipelineStages) + { + var kind = $"source:{sourceId}:{stage}"; + if (schedulerOptions.Definitions.TryGetValue(kind, out var definition) && definition.Enabled) + { + definitions.Add(definition); + } + } + + return definitions.ToImmutable(); + } + + private async Task ExecutePipelineAsync( + string sourceId, + string trigger, + JobRunSnapshot run, + ImmutableArray pipeline) + { + var stoppingToken = applicationLifetime.ApplicationStopping; + await concurrencyGate.WaitAsync(stoppingToken).ConfigureAwait(false); + + try + { + using var scope = serviceScopeFactory.CreateScope(); + + foreach (var definition in pipeline) + { + stoppingToken.ThrowIfCancellationRequested(); + + var job = (IJob)ActivatorUtilities.CreateInstance(scope.ServiceProvider, definition.JobType); + var jobLogger = loggerFactory.CreateLogger(definition.JobType); + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + if (definition.Timeout > TimeSpan.Zero) + { + timeoutCts.CancelAfter(definition.Timeout); + } + + var context = new JobExecutionContext( + run.RunId, + definition.Kind, + trigger, + EmptyParameters, + scope.ServiceProvider, + timeProvider, + jobLogger); + + logger.LogInformation( + "Running immediate source sync stage {JobKind} for source {SourceId}.", + definition.Kind, + sourceId); + + await job.ExecuteAsync(context, timeoutCts.Token).ConfigureAwait(false); + } + + logger.LogInformation( + "Completed immediate source sync pipeline for source {SourceId} with {StageCount} stage(s).", + sourceId, + pipeline.Length); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + logger.LogInformation("Immediate source sync pipeline for {SourceId} stopped during shutdown.", sourceId); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Immediate source sync pipeline for {SourceId} failed.", sourceId); + } + finally + { + activeRuns.TryRemove(sourceId, out _); + concurrencyGate.Release(); + } + } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobCoordinator.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobCoordinator.cs new file mode 100644 index 000000000..28dbc1a44 --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobCoordinator.cs @@ -0,0 +1,30 @@ +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.WebService.Services; + +public sealed class UnsupportedJobCoordinator : IJobCoordinator +{ + public const string Detail = + "Concelier job scheduling and run history require a durable backend implementation; the live runtime no longer falls back to in-memory job state."; + + public Task TriggerAsync(string kind, IReadOnlyDictionary? parameters, string trigger, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> GetDefinitionsAsync(CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task GetRunAsync(Guid runId, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobStore.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobStore.cs new file mode 100644 index 000000000..556fe81ae --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedJobStore.cs @@ -0,0 +1,33 @@ +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.WebService.Services; + +public sealed class UnsupportedJobStore : IJobStore +{ + public const string Detail = + "Concelier job run state requires a durable backend implementation; the live runtime no longer falls back to in-memory job storage."; + + public Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task FindAsync(Guid runId, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> GetActiveRunsAsync(CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedOrchestratorRegistryStore.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedOrchestratorRegistryStore.cs new file mode 100644 index 000000000..d4b229e2b --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/UnsupportedOrchestratorRegistryStore.cs @@ -0,0 +1,49 @@ +using StellaOps.Concelier.Core.Orchestration; + +namespace StellaOps.Concelier.WebService.Services; + +public sealed class UnsupportedOrchestratorRegistryStore : IOrchestratorRegistryStore +{ + public const string Detail = + "Concelier internal orchestrator registry and command state require a durable backend implementation; the live runtime no longer falls back to in-memory registry storage."; + + public Task UpsertAsync(OrchestratorRegistryRecord record, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task GetAsync(string tenant, string connectorId, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> ListAsync(string tenant, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task AppendHeartbeatAsync(OrchestratorHeartbeatRecord heartbeat, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task GetLatestHeartbeatAsync( + string tenant, + string connectorId, + Guid runId, + CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task EnqueueCommandAsync(OrchestratorCommandRecord command, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task> GetPendingCommandsAsync( + string tenant, + string connectorId, + Guid runId, + long? afterSequence, + CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task StoreManifestAsync(OrchestratorRunManifest manifest, CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); + + public Task GetManifestAsync( + string tenant, + string connectorId, + Guid runId, + CancellationToken cancellationToken) + => throw new NotSupportedException(Detail); +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/TASKS.md b/src/Concelier/StellaOps.Concelier.WebService/TASKS.md index c1621102c..22fb471ed 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/TASKS.md +++ b/src/Concelier/StellaOps.Concelier.WebService/TASKS.md @@ -9,4 +9,11 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0242-T | DONE | Revalidated 2026-01-07. | | AUDIT-0242-A | DONE | Applied 2026-01-13; TimeProvider defaults, ASCII cleanup, federation tests. | | BE8-07-API | DONE | Advisory-source freshness endpoint contract extended with advisory stats fields consumed by UI security diagnostics. | -| NOMOCK-003 | DOING | Replace seeded feed-mirror runtime endpoints with real source/read-model state and truthful unsupported responses. | +| NOMOCK-003 | DONE | 2026-04-18: Public mirror exports now require persisted mirror-domain state, `/api/v1/advisory-sources/mirror/import*` is backed by a durable filesystem import path instead of `501`, and the host now registers `IHttpContextAccessor` unconditionally for `ConcelierTopologyIdentityAccessor`. | +| NOMOCK-004 | DONE | 2026-04-18: Added `MirrorBundleImportRuntimeService` so live mirror import validates manifests/checksums/signatures, persists status, updates mirror-domain state, and publishes imported artifacts under `/concelier/exports/mirror/*`. | +| NOMOCK-026 | DONE | 2026-04-19: Hardened live mirror import with explicit `Mirror.ImportRoot` allowlisting so `bundlePath` and `trustRootsPath` cannot resolve outside the staged import root. Verified with focused Concelier rebuilds plus direct xUnit DLL coverage after `dotnet test --filter` proved unreliable in the current test platform path. | +| REALPLAN-007-C | DONE | 2026-04-15: Removed the live `InMemoryLeaseStore` binding; WebService now relies on the Postgres-backed lease store from `AddConcelierPostgresStorage`. | +| REALPLAN-007-D | DONE | 2026-04-19: Removed hidden live `InMemoryJobStore`/`InMemoryOrchestratorRegistryStore` fallbacks; `/jobs`, `/internal/orch/*`, and coordinator-backed manual sync compatibility routes now return explicit `501` until durable runtime storage exists. | +| REALPLAN-007-E | DONE | 2026-04-17: Removed the live `InMemoryAffectedSymbolStore` fallback; `/v1/signals/symbols/*` now returns explicit `501` outside `Testing` until a durable affected-symbol backend exists. | +| REALPLAN-007-F | DOING | 2026-04-19: Replacing the truthful `501` affected-symbol fallback with durable PostgreSQL-backed advisory-observation and affected-symbol runtime services fed from the real raw-ingest path. | +| ADV-SETUP-002 | DONE | `docs/implplan/SPRINT_20260417_001_Platform_setup_advisory_vex_onboarding.md`: made advisory source enablement/status durable and exposed setup bootstrap orchestration for source onboarding. | diff --git a/src/Concelier/StellaOps.Excititor.WebService/Options/GraphOptions.cs b/src/Concelier/StellaOps.Excititor.WebService/Options/GraphOptions.cs index c3e8308c3..a81b03427 100644 --- a/src/Concelier/StellaOps.Excititor.WebService/Options/GraphOptions.cs +++ b/src/Concelier/StellaOps.Excititor.WebService/Options/GraphOptions.cs @@ -8,7 +8,6 @@ public sealed class GraphOptions public int MaxPurls { get; set; } = 500; public int MaxAdvisoriesPerPurl { get; set; } = 200; public int OverlayTtlSeconds { get; set; } = 300; - public bool UsePostgresOverlayStore { get; set; } = true; public int MaxTooltipItemsPerPurl { get; set; } = 50; public int MaxTooltipTotal { get; set; } = 1000; } diff --git a/src/Concelier/StellaOps.Excititor.WebService/Program.cs b/src/Concelier/StellaOps.Excititor.WebService/Program.cs index 11098ddef..3a4dd8e6d 100644 --- a/src/Concelier/StellaOps.Excititor.WebService/Program.cs +++ b/src/Concelier/StellaOps.Excititor.WebService/Program.cs @@ -92,23 +92,11 @@ services.AddSingleton(); services.AddSingleton(); services.AddMemoryCache(); services.AddSingleton(); -services.AddSingleton(sp => -{ - var graphOptions = sp.GetRequiredService>().Value; - var pgOptions = sp.GetRequiredService>().Value; - if (graphOptions.UsePostgresOverlayStore && !string.IsNullOrWhiteSpace(pgOptions.ConnectionString)) - { - return new PostgresGraphOverlayStore( - sp.GetRequiredService(), - sp.GetRequiredService>()); - } - - return new InMemoryGraphOverlayStore(); -}); +services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); -// OBS-52/53/54: Attestation storage and timeline event recording -services.TryAddSingleton(); +// OBS-52/53/54: Attestation storage and timeline event recording. +// The durable Postgres store is registered by AddExcititorPersistence. services.TryAddSingleton(); services.AddScoped(); services.AddSingleton(); diff --git a/src/Concelier/StellaOps.Excititor.WebService/Services/IGraphOverlayStore.cs b/src/Concelier/StellaOps.Excititor.WebService/Services/IGraphOverlayStore.cs index 17734bac8..0debe909b 100644 --- a/src/Concelier/StellaOps.Excititor.WebService/Services/IGraphOverlayStore.cs +++ b/src/Concelier/StellaOps.Excititor.WebService/Services/IGraphOverlayStore.cs @@ -14,7 +14,7 @@ public interface IGraphOverlayStore } /// -/// In-memory overlay store placeholder until Postgres materialization is added. +/// Test/dev-only in-memory helper. Live hosts must bind . /// public sealed class InMemoryGraphOverlayStore : IGraphOverlayStore { diff --git a/src/Concelier/StellaOps.Excititor.WebService/TASKS.md b/src/Concelier/StellaOps.Excititor.WebService/TASKS.md index 271518b68..8e86e80bb 100644 --- a/src/Concelier/StellaOps.Excititor.WebService/TASKS.md +++ b/src/Concelier/StellaOps.Excititor.WebService/TASKS.md @@ -9,3 +9,5 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0327-T | DONE | Revalidated 2026-01-07; test coverage audit for Excititor.WebService. | | AUDIT-0327-A | TODO | Pending approval (non-test project; revalidated 2026-01-07). | | NOMOCK-012 | DONE | 2026-04-14: Removed live `InMemoryVexProviderStore`, `InMemoryVexConnectorStateRepository`, and `InMemoryVexClaimStore` fallbacks so the web runtime now resolves the persistence-backed Excititor stores. | +| REALPLAN-007-A | DONE | 2026-04-15: Removed the host-level `InMemoryVexAttestationStore` fallback; WebService now relies on the Postgres attestation store from `AddExcititorPersistence`, backed by durable attestation migration proof. | +| REALPLAN-007-B | DONE | 2026-04-15: Removed the live `InMemoryGraphOverlayStore` fallback; WebService now binds `IGraphOverlayStore` to `PostgresGraphOverlayStore` and startup migrations create `vex.graph_overlays`. | diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/ISourceSyncTrigger.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/ISourceSyncTrigger.cs new file mode 100644 index 000000000..0581a6c38 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/ISourceSyncTrigger.cs @@ -0,0 +1,14 @@ +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Core.Sources; + +/// +/// Triggers an initial source synchronization pipeline without depending on the generic jobs API surface. +/// +public interface ISourceSyncTrigger +{ + Task TriggerAsync( + string sourceId, + string trigger, + CancellationToken cancellationToken = default); +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs index e6314a816..43ca30b63 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs @@ -1386,9 +1386,9 @@ public static class SourceDefinitions Category = SourceCategory.Mirror, Type = SourceType.StellaMirror, Description = "StellaOps Pre-aggregated Advisory Mirror", - BaseEndpoint = "https://mirror.stella-ops.org/api/v1", - HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/stella-mirror", - HttpClientName = "StellaMirrorClient", + BaseEndpoint = "https://mirror.stella-ops.org", + HealthCheckEndpoint = "https://mirror.stella-ops.org/concelier/exports/index.json", + HttpClientName = "stellaops-mirror", RequiresAuthentication = false, // Can be configured for OAuth StatusPageUrl = "https://status.stella-ops.org/", DocumentationUrl = "https://docs.stella-ops.org/mirror/", diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs index b7f697a2d..ca4f1d92e 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs @@ -29,6 +29,7 @@ public sealed class SourceRegistry : ISourceRegistry private readonly TimeProvider _timeProvider; private readonly SourcesConfiguration _configuration; private readonly IJobCoordinator? _jobCoordinator; + private readonly ISourceSyncTrigger? _sourceSyncTrigger; private readonly ConcurrentDictionary _enabledSources; private readonly ConcurrentDictionary _lastCheckResults; @@ -37,13 +38,15 @@ public sealed class SourceRegistry : ISourceRegistry ILogger logger, TimeProvider? timeProvider = null, SourcesConfiguration? configuration = null, - IJobCoordinator? jobCoordinator = null) + IJobCoordinator? jobCoordinator = null, + ISourceSyncTrigger? sourceSyncTrigger = null) { _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _timeProvider = timeProvider ?? TimeProvider.System; _configuration = configuration ?? new SourcesConfiguration(); _jobCoordinator = jobCoordinator; + _sourceSyncTrigger = sourceSyncTrigger; _sources = SourceDefinitions.All; _enabledSources = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); _lastCheckResults = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); @@ -292,8 +295,27 @@ public sealed class SourceRegistry : ISourceRegistry _enabledSources[sourceId] = true; _logger.LogInformation("Enabled source: {SourceId}", sourceId); - // Auto-trigger initial fetch job on enable - if (_jobCoordinator is not null) + // Auto-trigger initial sync pipeline on enable. + if (_sourceSyncTrigger is not null) + { + try + { + var result = await _sourceSyncTrigger.TriggerAsync(sourceId, "source-enable", cancellationToken).ConfigureAwait(false); + if (result.Outcome == JobTriggerOutcome.Accepted) + { + _logger.LogInformation("Auto-triggered sync pipeline for source {SourceId} on enable", sourceId); + } + else + { + _logger.LogDebug("Sync pipeline for source {SourceId} was not started on enable: {Outcome}", sourceId, result.Outcome); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to auto-trigger sync pipeline for source {SourceId} on enable", sourceId); + } + } + else if (_jobCoordinator is not null) { var fetchKind = $"source:{sourceId}:fetch"; try diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/000_enable_pg_trgm_extension.sql b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/000_enable_pg_trgm_extension.sql new file mode 100644 index 000000000..551b15aea --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/000_enable_pg_trgm_extension.sql @@ -0,0 +1,5 @@ +-- Concelier/Vuln Schema: PostgreSQL extension prerequisites +-- Fresh blank databases must commit pg_trgm before the initial schema creates +-- trigram-backed GIN indexes such as gin_trgm_ops on advisory purl lookups. + +CREATE EXTENSION IF NOT EXISTS pg_trgm; diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/007_add_job_leases.sql b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/007_add_job_leases.sql new file mode 100644 index 000000000..1ef0e2b51 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/007_add_job_leases.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS vuln.job_leases ( + lease_key TEXT PRIMARY KEY, + holder TEXT NOT NULL, + acquired_at TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL, + lease_ms BIGINT NOT NULL CHECK (lease_ms > 0), + ttl_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_job_leases_ttl_at ON vuln.job_leases(ttl_at); +CREATE INDEX IF NOT EXISTS idx_job_leases_holder ON vuln.job_leases(holder); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresLeaseStore.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresLeaseStore.cs new file mode 100644 index 000000000..5c072cb11 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresLeaseStore.cs @@ -0,0 +1,142 @@ +using Microsoft.Extensions.Logging; +using Npgsql; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Infrastructure.Postgres.Repositories; + +namespace StellaOps.Concelier.Persistence.Postgres.Repositories; + +/// +/// Durable PostgreSQL-backed job lease store for Concelier orchestration. +/// +public sealed class PostgresLeaseStore : RepositoryBase, ILeaseStore +{ + private const string SystemTenantId = "_system"; + + public PostgresLeaseStore(ConcelierDataSource dataSource, ILogger logger) + : base(dataSource, logger) + { + } + + public Task TryAcquireAsync( + string key, + string holder, + TimeSpan leaseDuration, + DateTimeOffset now, + CancellationToken cancellationToken) + { + const string sql = """ + INSERT INTO vuln.job_leases + (lease_key, holder, acquired_at, heartbeat_at, lease_ms, ttl_at) + VALUES + (@lease_key, @holder, @acquired_at, @heartbeat_at, @lease_ms, @ttl_at) + ON CONFLICT (lease_key) DO UPDATE + SET holder = EXCLUDED.holder, + acquired_at = CASE + WHEN vuln.job_leases.holder = EXCLUDED.holder THEN vuln.job_leases.acquired_at + ELSE EXCLUDED.acquired_at + END, + heartbeat_at = EXCLUDED.heartbeat_at, + lease_ms = EXCLUDED.lease_ms, + ttl_at = EXCLUDED.ttl_at + WHERE vuln.job_leases.ttl_at <= @now OR vuln.job_leases.holder = EXCLUDED.holder + RETURNING lease_key, holder, acquired_at, heartbeat_at, lease_ms, ttl_at + """; + + var ttlAt = now.Add(leaseDuration); + var leaseMs = ToLeaseMilliseconds(leaseDuration); + + return QuerySingleOrDefaultAsync( + SystemTenantId, + sql, + cmd => + { + AddParameter(cmd, "lease_key", key); + AddParameter(cmd, "holder", holder); + AddParameter(cmd, "acquired_at", now); + AddParameter(cmd, "heartbeat_at", now); + AddParameter(cmd, "lease_ms", leaseMs); + AddParameter(cmd, "ttl_at", ttlAt); + AddParameter(cmd, "now", now); + }, + MapLease, + cancellationToken); + } + + public Task HeartbeatAsync( + string key, + string holder, + TimeSpan leaseDuration, + DateTimeOffset now, + CancellationToken cancellationToken) + { + const string sql = """ + UPDATE vuln.job_leases + SET heartbeat_at = @heartbeat_at, + lease_ms = @lease_ms, + ttl_at = @ttl_at + WHERE lease_key = @lease_key + AND holder = @holder + RETURNING lease_key, holder, acquired_at, heartbeat_at, lease_ms, ttl_at + """; + + var ttlAt = now.Add(leaseDuration); + var leaseMs = ToLeaseMilliseconds(leaseDuration); + + return QuerySingleOrDefaultAsync( + SystemTenantId, + sql, + cmd => + { + AddParameter(cmd, "lease_key", key); + AddParameter(cmd, "holder", holder); + AddParameter(cmd, "heartbeat_at", now); + AddParameter(cmd, "lease_ms", leaseMs); + AddParameter(cmd, "ttl_at", ttlAt); + }, + MapLease, + cancellationToken); + } + + public async Task ReleaseAsync(string key, string holder, CancellationToken cancellationToken) + { + const string sql = """ + DELETE FROM vuln.job_leases + WHERE lease_key = @lease_key + AND holder = @holder + """; + + var rows = await ExecuteAsync( + SystemTenantId, + sql, + cmd => + { + AddParameter(cmd, "lease_key", key); + AddParameter(cmd, "holder", holder); + }, + cancellationToken).ConfigureAwait(false); + + return rows > 0; + } + + private static JobLease MapLease(NpgsqlDataReader reader) + { + var leaseDuration = TimeSpan.FromMilliseconds(reader.GetInt64(4)); + return new JobLease( + reader.GetString(0), + reader.GetString(1), + reader.GetFieldValue(2), + reader.GetFieldValue(3), + leaseDuration, + reader.GetFieldValue(5)); + } + + private static long ToLeaseMilliseconds(TimeSpan leaseDuration) + { + if (leaseDuration <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(leaseDuration), "Lease duration must be positive."); + } + + return checked((long)Math.Ceiling(leaseDuration.TotalMilliseconds)); + } +} diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs index 12bdca8bc..536e094ef 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/ServiceCollectionExtensions.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using PsirtContracts = StellaOps.Concelier.Storage.PsirtFlags; using StellaOps.Concelier.Core.Canonical; +using StellaOps.Concelier.Core.Jobs; using StellaOps.Concelier.Core.Linksets; using StellaOps.Concelier.Merge.Backport; using StellaOps.Concelier.Persistence.Postgres.Advisories; @@ -76,6 +77,7 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); // Provenance scope services (backport integration) services.AddScoped(); @@ -134,6 +136,7 @@ public static class ServiceCollectionExtensions services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddSingleton(); // Provenance scope services (backport integration) services.AddScoped(); diff --git a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/TASKS.md b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/TASKS.md index 18b352312..669e65599 100644 --- a/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Concelier.Persistence/TASKS.md @@ -12,3 +12,5 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | TASK-015-011 | DONE | Added enriched SBOM storage table + Postgres repository. | | TASK-015-007d | DONE | Added license indexes and repository queries for license inventory. | | BE8-07 | DONE | Added migration `005_add_advisory_source_signature_projection.sql` and advisory-source signature stats projection fields for UI detail diagnostics. | +| REALPLAN-007-C | DONE | 2026-04-15: Added durable `PostgresLeaseStore` plus migration `007_add_job_leases.sql`; `AddConcelierPostgresStorage` now owns live Concelier lease coordination. | +| REALPLAN-007-F | DOING | 2026-04-19: Adding durable advisory-observation and affected-symbol persistence so live Concelier ingest can back `/v1/signals/symbols/*` with PostgreSQL instead of unsupported runtime fallbacks. | diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/004_graph_overlays.sql b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/004_graph_overlays.sql new file mode 100644 index 000000000..5cdbcdde7 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/004_graph_overlays.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS vex.graph_overlays ( + tenant text NOT NULL, + purl text NOT NULL, + advisory_id text NOT NULL, + source text NOT NULL, + generated_at timestamptz NOT NULL, + payload jsonb NOT NULL, + CONSTRAINT pk_graph_overlays PRIMARY KEY (tenant, purl, advisory_id, source) +); + +CREATE INDEX IF NOT EXISTS idx_graph_overlays_generated_at + ON vex.graph_overlays (tenant, generated_at DESC); diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/005_vex_attestations.sql b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/005_vex_attestations.sql new file mode 100644 index 000000000..f34804bc7 --- /dev/null +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Migrations/005_vex_attestations.sql @@ -0,0 +1,46 @@ +-- Migration: 005_vex_attestations +-- Category: startup +-- Description: Create durable storage for VEX attestations. + +CREATE TABLE IF NOT EXISTS attestations ( + tenant TEXT NOT NULL, + attestation_id TEXT NOT NULL, + manifest_id TEXT NOT NULL, + merkle_root TEXT NOT NULL, + dsse_envelope_json TEXT NOT NULL, + dsse_envelope_hash TEXT NOT NULL, + item_count INTEGER NOT NULL, + attested_at TIMESTAMPTZ NOT NULL, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT attestations_pkey PRIMARY KEY (tenant, attestation_id) +); + +CREATE INDEX IF NOT EXISTS idx_attestations_tenant + ON attestations (tenant); + +CREATE INDEX IF NOT EXISTS idx_attestations_manifest_id + ON attestations (tenant, manifest_id); + +CREATE INDEX IF NOT EXISTS idx_attestations_attested_at + ON attestations (tenant, attested_at DESC); + +ALTER TABLE attestations ENABLE ROW LEVEL SECURITY; +ALTER TABLE attestations FORCE ROW LEVEL SECURITY; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = current_schema() + AND tablename = 'attestations' + AND policyname = 'attestations_tenant_isolation' + ) THEN + CREATE POLICY attestations_tenant_isolation ON attestations + FOR ALL + USING (tenant = vex_app.require_current_tenant()) + WITH CHECK (tenant = vex_app.require_current_tenant()); + END IF; +END +$$; diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexAttestationStore.cs b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexAttestationStore.cs index 261c2303b..84f372499 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexAttestationStore.cs +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/Postgres/Repositories/PostgresVexAttestationStore.cs @@ -236,7 +236,15 @@ public sealed class PostgresVexAttestationStore : RepositoryBase ExcititorDataSource.DefaultSchemaName; + private string GetSchemaName() + { + if (!string.IsNullOrWhiteSpace(DataSource.SchemaName)) + { + return DataSource.SchemaName; + } + + return ExcititorDataSource.DefaultSchemaName; + } private static bool IsUniqueViolation(DbUpdateException exception) { diff --git a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/TASKS.md b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/TASKS.md index 7cd502a1d..a26a71146 100644 --- a/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/TASKS.md +++ b/src/Concelier/__Libraries/StellaOps.Excititor.Persistence/TASKS.md @@ -11,3 +11,4 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | VEX-LINK-STORE-0001 | DONE | SPRINT_20260113_003_001 - Evidence link migration added. | | QA-DEVOPS-VERIFY-002-F | DONE | 2026-02-11: Fixed Rekor-linkage schema mismatch in `PostgresVexObservationStore` by aligning to `vex.observations` and ensuring Rekor linkage columns/indexes. | | NOMOCK-012 | DONE | 2026-04-14: Added PostgreSQL `IVexClaimStore`, wired startup migrations for `vex`, removed live demo-seed SQL, and restricted embedded Excititor migrations to active top-level files only. | +| REALPLAN-007-A | DONE | 2026-04-15: Added startup migration `005_vex_attestations.sql` so `PostgresVexAttestationStore` owns a durable attestation table in the active runtime schema. | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Sources/SourceRegistryTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Sources/SourceRegistryTests.cs index 002ae9861..c052e08e6 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Sources/SourceRegistryTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Core.Tests/Sources/SourceRegistryTests.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using Moq; using Moq.Protected; +using StellaOps.Concelier.Core.Jobs; using StellaOps.Concelier.Core.Configuration; using StellaOps.Concelier.Core.Sources; using Xunit; @@ -165,6 +166,44 @@ public sealed class SourceRegistryTests Assert.False(result); } + [Fact] + public async Task EnableSourceAsync_TriggersImmediateSync_WhenTriggerRegistered() + { + var syncTrigger = new Mock(); + syncTrigger + .Setup(trigger => trigger.TriggerAsync("nvd", "source-enable", It.IsAny())) + .ReturnsAsync(JobTriggerResult.Accepted(new JobRunSnapshot( + Guid.NewGuid(), + "source:nvd:fetch", + JobRunStatus.Pending, + FixedNow, + FixedNow, + null, + "source-enable", + null, + null, + null, + null, + new Dictionary()))); + + var registry = new SourceRegistry( + _httpClientFactoryMock.Object, + NullLogger.Instance, + _timeProvider, + configuration: null, + jobCoordinator: null, + sourceSyncTrigger: syncTrigger.Object); + + await registry.DisableSourceAsync("nvd", TestContext.Current.CancellationToken); + + var result = await registry.EnableSourceAsync("nvd", TestContext.Current.CancellationToken); + + Assert.True(result); + syncTrigger.Verify( + trigger => trigger.TriggerAsync("nvd", "source-enable", It.IsAny()), + Times.Once); + } + [Fact] public async Task DisableSourceAsync_DisablesSource_ReturnsTrue() { diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierInfrastructureRegistrationTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierInfrastructureRegistrationTests.cs index ef1898989..2c113d54d 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierInfrastructureRegistrationTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/ConcelierInfrastructureRegistrationTests.cs @@ -2,7 +2,9 @@ using FluentAssertions; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using StellaOps.Concelier.Core.Jobs; using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.Concelier.Persistence.Postgres.Repositories; using StellaOps.TestKit; namespace StellaOps.Concelier.Persistence.Tests; @@ -30,4 +32,26 @@ public sealed class ConcelierInfrastructureRegistrationTests .Should() .ContainSingle("fresh installs need Concelier startup migrations to create the vuln schema before canonical advisory queries can execute"); } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public void AddConcelierPostgresStorage_RegistersPostgresLeaseStore() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Postgres:Concelier:ConnectionString"] = "Host=postgres;Database=stellaops;Username=postgres;Password=postgres" + }) + .Build(); + + var services = new ServiceCollection(); + services.AddLogging(); + + services.AddConcelierPostgresStorage(configuration); + + services.Should().Contain(descriptor => + descriptor.ServiceType == typeof(ILeaseStore) + && descriptor.ImplementationType == typeof(PostgresLeaseStore), + "live Concelier job coordination must use the durable PostgreSQL lease store"); + } } diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresLeaseStoreTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresLeaseStoreTests.cs new file mode 100644 index 000000000..af3cc0873 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/PostgresLeaseStoreTests.cs @@ -0,0 +1,150 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.Concelier.Persistence.Postgres.Repositories; +using Xunit; + +using StellaOps.TestKit; +namespace StellaOps.Concelier.Persistence.Tests; + +[Collection(ConcelierPostgresCollection.Name)] +public sealed class PostgresLeaseStoreTests : IAsyncLifetime +{ + private readonly ConcelierPostgresFixture _fixture; + private readonly ConcelierDataSource _dataSource; + private readonly PostgresLeaseStore _store; + + public PostgresLeaseStoreTests(ConcelierPostgresFixture fixture) + { + _fixture = fixture; + _dataSource = CreateDataSource(); + _store = new PostgresLeaseStore(_dataSource, NullLogger.Instance); + } + + public ValueTask InitializeAsync() => new(_fixture.TruncateAllTablesAsync()); + + public async ValueTask DisposeAsync() + { + await _dataSource.DisposeAsync(); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task TryAcquireAsync_RejectsCompetingHolderUntilLeaseExpiresAcrossStoreInstances() + { + var now = new DateTimeOffset(2026, 4, 15, 10, 0, 0, TimeSpan.Zero); + + var firstLease = await _store.TryAcquireAsync( + "jobs:nvd:fetch", + "node-a", + TimeSpan.FromMinutes(5), + now, + CancellationToken.None); + + firstLease.Should().NotBeNull(); + + await using var restartedDataSource = CreateDataSource(); + var restartedStore = new PostgresLeaseStore(restartedDataSource, NullLogger.Instance); + + var competingLease = await restartedStore.TryAcquireAsync( + "jobs:nvd:fetch", + "node-b", + TimeSpan.FromMinutes(5), + now.AddMinutes(1), + CancellationToken.None); + + competingLease.Should().BeNull("an active persisted lease must block other nodes after restart"); + + var expiredLease = await restartedStore.TryAcquireAsync( + "jobs:nvd:fetch", + "node-b", + TimeSpan.FromMinutes(5), + now.AddMinutes(6), + CancellationToken.None); + + expiredLease.Should().NotBeNull(); + expiredLease!.Holder.Should().Be("node-b"); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task HeartbeatAsync_ExtendsLeaseAndPreservesOriginalAcquiredAt() + { + var acquiredAt = new DateTimeOffset(2026, 4, 15, 10, 0, 0, TimeSpan.Zero); + var initialLease = await _store.TryAcquireAsync( + "jobs:ghsa:sync", + "node-a", + TimeSpan.FromMinutes(4), + acquiredAt, + CancellationToken.None); + + initialLease.Should().NotBeNull(); + + var heartbeatAt = acquiredAt.AddMinutes(2); + var renewedLease = await _store.HeartbeatAsync( + "jobs:ghsa:sync", + "node-a", + TimeSpan.FromMinutes(4), + heartbeatAt, + CancellationToken.None); + + renewedLease.Should().NotBeNull(); + renewedLease!.AcquiredAt.Should().Be(acquiredAt); + renewedLease.HeartbeatAt.Should().Be(heartbeatAt); + renewedLease.TtlAt.Should().Be(heartbeatAt.AddMinutes(4)); + + await using var competingDataSource = CreateDataSource(); + var competingStore = new PostgresLeaseStore(competingDataSource, NullLogger.Instance); + + var competingLease = await competingStore.TryAcquireAsync( + "jobs:ghsa:sync", + "node-b", + TimeSpan.FromMinutes(4), + acquiredAt.AddMinutes(5), + CancellationToken.None); + + competingLease.Should().BeNull("heartbeat should extend the persisted TTL for competing holders"); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task ReleaseAsync_DeletesOwnedLeaseAndAllowsImmediateReacquire() + { + var now = new DateTimeOffset(2026, 4, 15, 12, 0, 0, TimeSpan.Zero); + var initialLease = await _store.TryAcquireAsync( + "jobs:redhat:map", + "node-a", + TimeSpan.FromMinutes(3), + now, + CancellationToken.None); + + initialLease.Should().NotBeNull(); + + var wrongHolderRelease = await _store.ReleaseAsync("jobs:redhat:map", "node-b", CancellationToken.None); + wrongHolderRelease.Should().BeFalse(); + + var released = await _store.ReleaseAsync("jobs:redhat:map", "node-a", CancellationToken.None); + released.Should().BeTrue(); + + await using var restartedDataSource = CreateDataSource(); + var restartedStore = new PostgresLeaseStore(restartedDataSource, NullLogger.Instance); + + var reacquired = await restartedStore.TryAcquireAsync( + "jobs:redhat:map", + "node-b", + TimeSpan.FromMinutes(3), + now.AddSeconds(1), + CancellationToken.None); + + reacquired.Should().NotBeNull("release must remove the persisted row so another node can take the lease"); + reacquired!.Holder.Should().Be("node-b"); + } + + private ConcelierDataSource CreateDataSource() + { + var options = _fixture.CreateOptions(); + return new ConcelierDataSource(Options.Create(options), NullLogger.Instance); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/TASKS.md index 9c1325a37..184c806ed 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.Persistence.Tests/TASKS.md @@ -12,3 +12,5 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | TASK-015-007d | DONE | Added license query coverage for SbomRepository. | | TASK-015-013 | DONE | Added SbomRepository integration coverage for model cards and policy fields. | | TASK-014-003 | DONE | 2026-03-09: added startup-migration registration coverage so Concelier canonical tables bootstrap on fresh deploys and verified `/api/v1/canonical` live after redeploy. | +| REALPLAN-007-C | DONE | 2026-04-15: Added `PostgresLeaseStoreTests` and DI registration coverage for durable lease coordination in `vuln.job_leases`. | +| REALPLAN-007-F | DOING | 2026-04-19: Adding persistence coverage for durable advisory observations, affected-symbol storage, and ingest-time symbol extraction against PostgreSQL. | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySourceEndpointsTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySourceEndpointsTests.cs index 6b37ad62d..6f969b503 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySourceEndpointsTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/AdvisorySourceEndpointsTests.cs @@ -4,15 +4,18 @@ using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Http; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System.Net; using System.Security.Claims; +using System.Security.Authentication; using System.Net.Http.Json; using System.Text.Json; using System.Text.Encodings.Web; using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Core.Sources; using StellaOps.Concelier.Persistence.Postgres.Models; using StellaOps.Concelier.Persistence.Postgres.Repositories; using StellaOps.Concelier.WebService.Extensions; @@ -24,8 +27,28 @@ namespace StellaOps.Concelier.WebService.Tests; public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory { + private static readonly SourceEntity BaselineNvdSource = new() + { + Id = Guid.Parse("0e94e643-4045-4af8-b0c4-f4f04f769279"), + Key = "nvd", + Name = "NVD", + SourceType = "nvd", + Url = "https://nvd.nist.gov", + Priority = 100, + Enabled = true, + Config = """{"syncIntervalMinutes":360,"localPath":"/data/mirrors/nvd"}""", + CreatedAt = DateTimeOffset.Parse("2026-02-18T00:00:00Z"), + UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z") + }; + private readonly StubJobCoordinator _jobCoordinator = new(); + private readonly StubSourceSyncTrigger _sourceSyncTrigger = new(); + private readonly StubSourceRepository _sourceRepository = new(); + private readonly InMemoryMirrorConfigStore _mirrorConfigStore = new(); + private readonly InMemoryMirrorConsumerConfigStore _mirrorConsumerConfigStore = new(); private readonly string _mirrorExportRoot = Directory.CreateTempSubdirectory("concelier-feedmirror-tests-").FullName; + private Func _mirrorProbeResponseFactory = + _ => new HttpResponseMessage(HttpStatusCode.OK); public AdvisorySourceWebAppFactory() { @@ -43,6 +66,7 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory Environment.SetEnvironmentVariable("CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK", "false"); Environment.SetEnvironmentVariable("CONCELIER_AUTHORITY__REQUIREHTTPSMETADATA", "false"); Environment.SetEnvironmentVariable("CONCELIER_AUTHORITY__ISSUER", "https://authority.test"); + Environment.SetEnvironmentVariable("STELLAOPS_BOOTSTRAP_KEY", "test-bootstrap-key"); } protected override void ConfigureWebHost(IWebHostBuilder builder) @@ -83,6 +107,7 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory "Concelier.Jobs.Trigger", "Concelier.Observations.Read", "Concelier.Aoc.Verify", "Concelier.Canonical.Read", "Concelier.Canonical.Ingest", "Concelier.Interest.Read", + "Concelier.Sources.Manage", "Concelier.Interest.Admin", }) { @@ -97,13 +122,31 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory services.AddSingleton(); services.RemoveAll(); - services.AddSingleton(); + services.AddSingleton(_sourceRepository); + services.AddSingleton(sp => sp.GetRequiredService()); + services.RemoveAll(); + services.AddSingleton(_mirrorConfigStore); + services.AddSingleton(sp => sp.GetRequiredService()); + services.RemoveAll(); + services.AddSingleton(_mirrorConsumerConfigStore); + services.AddSingleton(sp => sp.GetRequiredService()); + services.Configure("MirrorConsumer", options => + { + options.HttpMessageHandlerBuilderActions.Clear(); + options.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = new MirrorProbeHttpMessageHandler(() => _mirrorProbeResponseFactory); + }); + }); services.RemoveAll(); services.AddSingleton(); services.AddSingleton(_jobCoordinator); services.RemoveAll(); services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(_sourceSyncTrigger); + services.RemoveAll(); + services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(new ConcelierOptions { @@ -151,9 +194,146 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory }); } - public string? LastTriggeredKind => _jobCoordinator.LastTriggeredKind; + public string? LastTriggeredKind => _sourceSyncTrigger.LastTriggeredKind ?? _jobCoordinator.LastTriggeredKind; - public void ResetTriggeredJobs() => _jobCoordinator.Reset(); + public SourceEntity? GetSource(string key) => _sourceRepository.GetSnapshot(key); + + public MirrorConfigRecord GetMirrorConfig() => _mirrorConfigStore.GetConfig(); + + public ConsumerConfigResponse GetMirrorConsumerConfig() => _mirrorConsumerConfigStore.GetConsumerConfig(); + + public void ResetTriggeredJobs() + { + _jobCoordinator.Reset(); + _sourceSyncTrigger.Reset(); + } + + public void ResetState() + { + ResetTriggeredJobs(); + _sourceRepository.Reset(); + _mirrorConfigStore.Reset(); + _mirrorConsumerConfigStore.Reset(); + _mirrorProbeResponseFactory = _ => new HttpResponseMessage(HttpStatusCode.OK); + } + + public void SetMirrorProbeResponseFactory(Func responseFactory) + { + _mirrorProbeResponseFactory = responseFactory ?? throw new ArgumentNullException(nameof(responseFactory)); + } + + private sealed class MirrorProbeHttpMessageHandler : HttpMessageHandler + { + private readonly Func> _factoryAccessor; + + public MirrorProbeHttpMessageHandler(Func> factoryAccessor) + { + _factoryAccessor = factoryAccessor; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(_factoryAccessor()(request)); + } + + private sealed class InMemoryMirrorConfigStore : IMirrorConfigStore + { + private readonly object syncRoot = new(); + private MirrorConfigRecord config = new(); + + public MirrorConfigRecord GetConfig() + { + lock (syncRoot) + { + return new MirrorConfigRecord + { + Mode = config.Mode, + OutputRoot = config.OutputRoot, + ConsumerBaseAddress = config.ConsumerBaseAddress, + SigningEnabled = config.SigningEnabled, + SigningAlgorithm = config.SigningAlgorithm, + SigningKeyId = config.SigningKeyId, + AutoRefreshEnabled = config.AutoRefreshEnabled, + RefreshIntervalMinutes = config.RefreshIntervalMinutes, + }; + } + } + + public Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + lock (syncRoot) + { + config = new MirrorConfigRecord + { + Mode = request.Mode ?? config.Mode, + OutputRoot = config.OutputRoot, + ConsumerBaseAddress = request.ConsumerBaseAddress ?? config.ConsumerBaseAddress, + SigningEnabled = request.Signing?.Enabled ?? config.SigningEnabled, + SigningAlgorithm = request.Signing?.Algorithm ?? config.SigningAlgorithm, + SigningKeyId = request.Signing?.KeyId ?? config.SigningKeyId, + AutoRefreshEnabled = request.AutoRefreshEnabled ?? config.AutoRefreshEnabled, + RefreshIntervalMinutes = request.RefreshIntervalMinutes ?? config.RefreshIntervalMinutes, + }; + } + + return Task.CompletedTask; + } + + public void Reset() + { + lock (syncRoot) + { + config = new MirrorConfigRecord(); + } + } + } + + private sealed class InMemoryMirrorConsumerConfigStore : IMirrorConsumerConfigStore + { + private readonly object syncRoot = new(); + private ConsumerConfigResponse config = new() + { + Connected = false, + }; + + public ConsumerConfigResponse GetConsumerConfig() + { + lock (syncRoot) + { + return config; + } + } + + public Task SetConsumerConfigAsync(ConsumerConfigRequest request, CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + lock (syncRoot) + { + config = new ConsumerConfigResponse + { + BaseAddress = request.BaseAddress, + DomainId = request.DomainId, + IndexPath = string.IsNullOrWhiteSpace(request.IndexPath) ? "/concelier/exports/index.json" : request.IndexPath, + HttpTimeoutSeconds = request.HttpTimeoutSeconds ?? 30, + Connected = true, + LastSync = DateTimeOffset.UtcNow, + }; + } + + return Task.CompletedTask; + } + + public void Reset() + { + lock (syncRoot) + { + config = new ConsumerConfigResponse + { + Connected = false, + }; + } + } + } private sealed class TestAuthHandler : AuthenticationHandler { @@ -251,51 +431,61 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory private sealed class StubSourceRepository : ISourceRepository { - private static readonly List Sources = - [ - new SourceEntity - { - Id = Guid.Parse("0e94e643-4045-4af8-b0c4-f4f04f769279"), - Key = "nvd", - Name = "NVD", - SourceType = "nvd", - Url = "https://nvd.nist.gov", - Priority = 100, - Enabled = true, - Config = """{"syncIntervalMinutes":360,"localPath":"/data/mirrors/nvd"}""", - CreatedAt = DateTimeOffset.Parse("2026-02-18T00:00:00Z"), - UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z") - } - ]; + private readonly List _sources = [Clone(AdvisorySourceWebAppFactory.BaselineNvdSource)]; public Task UpsertAsync(SourceEntity source, CancellationToken cancellationToken = default) { - var index = Sources.FindIndex(existing => existing.Id == source.Id); + var index = _sources.FindIndex(existing => existing.Id == source.Id || string.Equals(existing.Key, source.Key, StringComparison.OrdinalIgnoreCase)); if (index >= 0) { - Sources[index] = source; + _sources[index] = source; } else { - Sources.Add(source); + _sources.Add(source); } return Task.FromResult(source); } public Task GetByIdAsync(Guid id, CancellationToken cancellationToken = default) - => Task.FromResult(Sources.FirstOrDefault(source => source.Id == id)); + => Task.FromResult(_sources.FirstOrDefault(source => source.Id == id)); public Task GetByKeyAsync(string key, CancellationToken cancellationToken = default) - => Task.FromResult(Sources.FirstOrDefault(source => string.Equals(source.Key, key, StringComparison.OrdinalIgnoreCase))); + => Task.FromResult(_sources.FirstOrDefault(source => string.Equals(source.Key, key, StringComparison.OrdinalIgnoreCase))); public Task> ListAsync(bool? enabled = null, CancellationToken cancellationToken = default) { - var items = Sources + var items = _sources .Where(source => enabled is null || source.Enabled == enabled.Value) .ToList(); return Task.FromResult>(items); } + + public SourceEntity? GetSnapshot(string key) => + _sources.FirstOrDefault(source => string.Equals(source.Key, key, StringComparison.OrdinalIgnoreCase)); + + public void Reset() + { + _sources.Clear(); + _sources.Add(Clone(AdvisorySourceWebAppFactory.BaselineNvdSource)); + } + + private static SourceEntity Clone(SourceEntity source) => + new() + { + Id = source.Id, + Key = source.Key, + Name = source.Name, + SourceType = source.SourceType, + Url = source.Url, + Priority = source.Priority, + Enabled = source.Enabled, + Config = source.Config, + Metadata = source.Metadata, + CreatedAt = source.CreatedAt, + UpdatedAt = source.UpdatedAt + }; } private sealed class StubFeedSnapshotRepository : IFeedSnapshotRepository @@ -398,6 +588,33 @@ public sealed class AdvisorySourceWebAppFactory : WebApplicationFactory public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) => Task.FromResult>(new Dictionary(StringComparer.OrdinalIgnoreCase)); } + + private sealed class StubSourceSyncTrigger : ISourceSyncTrigger + { + public string? LastTriggeredKind { get; private set; } + + public void Reset() => LastTriggeredKind = null; + + public Task TriggerAsync(string sourceId, string trigger, CancellationToken cancellationToken = default) + { + var kind = $"source:{sourceId}:fetch"; + LastTriggeredKind = kind; + var now = DateTimeOffset.Parse("2026-02-19T08:30:00Z"); + return Task.FromResult(JobTriggerResult.Accepted(new JobRunSnapshot( + Guid.Parse("641c0427-4d8f-4f88-9496-b6a691037faa"), + kind, + JobRunStatus.Pending, + now, + now, + null, + trigger, + null, + null, + null, + null, + new Dictionary()))); + } + } } public sealed class AdvisorySourceEndpointsTests : IClassFixture @@ -411,13 +628,17 @@ public sealed class AdvisorySourceEndpointsTests : IClassFixture(cancellationToken: CancellationToken.None); + Assert.NotNull(payload); + Assert.NotEmpty(payload!.Items); } [Trait("Category", TestCategories.Unit)] @@ -458,6 +679,124 @@ public sealed class AdvisorySourceEndpointsTests : IClassFixture(cancellationToken: CancellationToken.None); + Assert.NotNull(payload); + + var nvd = Assert.Single(payload!.Sources.Where(source => source.SourceId == "nvd")); + Assert.True(nvd.Enabled); + + var osv = Assert.Single(payload.Sources.Where(source => source.SourceId == "osv")); + Assert.False(osv.Enabled); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task EnableEndpoint_PersistsMirrorAndTriggersInitialAggregation() + { + _factory.ResetState(); + _factory.SetMirrorProbeResponseFactory(_ => new HttpResponseMessage(HttpStatusCode.OK)); + _factory.ResetTriggeredJobs(); + using var client = CreateTenantClient(); + + var response = await client.PostAsync("/api/v1/advisory-sources/stella-mirror/enable", content: null, CancellationToken.None); + + response.EnsureSuccessStatusCode(); + var persisted = _factory.GetSource("stella-mirror"); + Assert.NotNull(persisted); + Assert.True(persisted!.Enabled); + Assert.Equal("Mirror", _factory.GetMirrorConfig().Mode); + Assert.Equal("https://mirror.stella-ops.org", _factory.GetMirrorConsumerConfig().BaseAddress); + Assert.Equal("source:stella-mirror:fetch", _factory.LastTriggeredKind); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task InternalSetupApply_EnablesMirrorAndDisablesPreviouslyConfiguredSources() + { + _factory.ResetState(); + _factory.SetMirrorProbeResponseFactory(_ => new HttpResponseMessage(HttpStatusCode.OK)); + _factory.ResetTriggeredJobs(); + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-StellaOps-Bootstrap-Key", "test-bootstrap-key"); + + var response = await client.PostAsJsonAsync("/internal/setup/advisory-sources/apply", new + { + configValues = new Dictionary + { + ["sources.mode"] = "mirror", + ["sources.mirror.url"] = "https://mirror.stella-ops.org/api/v1" + } + }, CancellationToken.None); + + response.EnsureSuccessStatusCode(); + using var document = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(CancellationToken.None), cancellationToken: CancellationToken.None); + Assert.True(document.RootElement.GetProperty("applied").GetBoolean()); + Assert.True(document.RootElement.GetProperty("connectivityReady").GetBoolean()); + + var mirror = _factory.GetSource("stella-mirror"); + var nvd = _factory.GetSource("nvd"); + Assert.NotNull(mirror); + Assert.True(mirror!.Enabled); + Assert.Equal("https://mirror.stella-ops.org", mirror.Url); + Assert.NotNull(nvd); + Assert.False(nvd!.Enabled); + Assert.Equal("Mirror", _factory.GetMirrorConfig().Mode); + Assert.Equal("https://mirror.stella-ops.org", _factory.GetMirrorConsumerConfig().BaseAddress); + Assert.Equal("primary", _factory.GetMirrorConsumerConfig().DomainId); + Assert.Equal("source:stella-mirror:fetch", _factory.LastTriggeredKind); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task InternalSetupApply_DoesNotPersistMirror_WhenMirrorConnectivityFails() + { + _factory.ResetState(); + _factory.SetMirrorProbeResponseFactory(_ => + throw new HttpRequestException( + "The SSL connection could not be established, see inner exception.", + new AuthenticationException("RemoteCertificateNameMismatch"))); + try + { + _factory.ResetTriggeredJobs(); + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-StellaOps-Bootstrap-Key", "test-bootstrap-key"); + + var response = await client.PostAsJsonAsync("/internal/setup/advisory-sources/apply", new + { + configValues = new Dictionary + { + ["sources.mode"] = "mirror", + ["sources.mirror.url"] = "https://mirror.stella-ops.org" + } + }, CancellationToken.None); + + response.EnsureSuccessStatusCode(); + + using var document = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync(CancellationToken.None), cancellationToken: CancellationToken.None); + Assert.False(document.RootElement.GetProperty("applied").GetBoolean()); + Assert.False(document.RootElement.GetProperty("connectivityReady").GetBoolean()); + Assert.Contains( + "hostname-valid certificate", + document.RootElement.GetProperty("connectivityMessage").GetString(), + StringComparison.OrdinalIgnoreCase); + Assert.Null(_factory.GetSource("stella-mirror")); + Assert.Null(_factory.LastTriggeredKind); + } + finally + { + _factory.SetMirrorProbeResponseFactory(_ => new HttpResponseMessage(HttpStatusCode.OK)); + } + } + [Trait("Category", TestCategories.Unit)] [Fact] public async Task FreshnessEndpoint_ByKey_ReturnsRecord() diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierOptionsPostConfigureTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierOptionsPostConfigureTests.cs index bc8e26df8..d1a347e9d 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierOptionsPostConfigureTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierOptionsPostConfigureTests.cs @@ -65,4 +65,34 @@ public sealed class ConcelierOptionsPostConfigureTests Assert.Contains("Authority client secret file", exception.Message); } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public void Apply_ResolvesMirrorImportRootAbsolute() + { + var tempDirectory = Directory.CreateTempSubdirectory(); + try + { + var options = new ConcelierOptions + { + Mirror = new ConcelierOptions.MirrorOptions + { + ImportRoot = Path.Combine("incoming", "mirror"), + } + }; + + ConcelierOptionsPostConfigure.Apply(options, tempDirectory.FullName); + + Assert.Equal( + Path.Combine(tempDirectory.FullName, "incoming", "mirror"), + options.Mirror.ImportRootAbsolute); + } + finally + { + if (Directory.Exists(tempDirectory.FullName)) + { + Directory.Delete(tempDirectory.FullName, recursive: true); + } + } + } } diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierOptionsValidatorTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierOptionsValidatorTests.cs index 908aaff4c..67d1fae30 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierOptionsValidatorTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ConcelierOptionsValidatorTests.cs @@ -24,6 +24,8 @@ public sealed class ConcelierOptionsValidatorTests Enabled = true, ExportRoot = "/var/lib/concelier/jobs/mirror-exports", ExportRootAbsolute = "/var/lib/concelier/jobs/mirror-exports", + ImportRoot = "/var/lib/concelier/jobs/mirror-imports", + ImportRootAbsolute = "/var/lib/concelier/jobs/mirror-imports", LatestDirectoryName = "latest", MirrorDirectoryName = "mirror" }, diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ImmediateSourceSyncTriggerTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ImmediateSourceSyncTriggerTests.cs new file mode 100644 index 000000000..c4cc7b5d8 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/ImmediateSourceSyncTriggerTests.cs @@ -0,0 +1,181 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Core.Sources; +using StellaOps.Concelier.WebService.Services; +using Xunit; + +namespace StellaOps.Concelier.WebService.Tests; + +public sealed class ImmediateSourceSyncTriggerTests +{ + [Fact] + public async Task TriggerAsync_RunsRegisteredPipelineInOrder() + { + var recorder = new PipelineRecorder(expectedSteps: 3); + using var services = BuildServices(recorder, options => + { + options.MaxConcurrentJobs = 1; + options.Definitions["source:stella-mirror:fetch"] = CreateDefinition(typeof(FetchJob), "source:stella-mirror:fetch"); + options.Definitions["source:stella-mirror:parse"] = CreateDefinition(typeof(ParseJob), "source:stella-mirror:parse"); + options.Definitions["source:stella-mirror:map"] = CreateDefinition(typeof(MapJob), "source:stella-mirror:map"); + }); + + var trigger = services.GetRequiredService(); + + var result = await trigger.TriggerAsync("stella-mirror", "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + await recorder.Completed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(["fetch", "parse", "map"], recorder.Steps); + } + + [Fact] + public async Task TriggerAsync_CreatesPipelineJobs_FromDefinitionsWithoutExplicitRegistrations() + { + var recorder = new PipelineRecorder(expectedSteps: 3); + using var services = BuildServices(recorder, options => + { + options.MaxConcurrentJobs = 1; + options.Definitions["source:stella-mirror:fetch"] = CreateDefinition(typeof(FetchJob), "source:stella-mirror:fetch"); + options.Definitions["source:stella-mirror:parse"] = CreateDefinition(typeof(ParseJob), "source:stella-mirror:parse"); + options.Definitions["source:stella-mirror:map"] = CreateDefinition(typeof(MapJob), "source:stella-mirror:map"); + }, registerJobs: false); + + var trigger = services.GetRequiredService(); + + var result = await trigger.TriggerAsync("stella-mirror", "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, result.Outcome); + await recorder.Completed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(["fetch", "parse", "map"], recorder.Steps); + } + + [Fact] + public async Task TriggerAsync_ReturnsAlreadyRunning_WhenSourcePipelineIsInFlight() + { + var recorder = new BlockingRecorder(); + using var services = BuildServices(recorder, options => + { + options.MaxConcurrentJobs = 1; + options.Definitions["source:stella-mirror:fetch"] = CreateDefinition(typeof(BlockingFetchJob), "source:stella-mirror:fetch"); + }); + + var trigger = services.GetRequiredService(); + + var first = await trigger.TriggerAsync("stella-mirror", "unit-test", CancellationToken.None); + var second = await trigger.TriggerAsync("stella-mirror", "unit-test", CancellationToken.None); + + Assert.Equal(JobTriggerOutcome.Accepted, first.Outcome); + Assert.Equal(JobTriggerOutcome.AlreadyRunning, second.Outcome); + + recorder.Release.SetResult(true); + await recorder.Completed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + } + + private static ServiceProvider BuildServices( + TRecorder recorder, + Action configure, + bool registerJobs = true) + where TRecorder : class + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddSingleton(TimeProvider.System); + services.AddSingleton(); + services.AddSingleton(recorder); + if (registerJobs) + { + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + } + services.Configure(configure); + services.AddSingleton(); + return services.BuildServiceProvider(); + } + + private static JobDefinition CreateDefinition(Type jobType, string kind) => + new( + kind, + jobType, + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(5), + null, + true); + + private sealed class PipelineRecorder + { + private readonly int expectedSteps; + + public PipelineRecorder(int expectedSteps) + { + this.expectedSteps = expectedSteps; + } + + public List Steps { get; } = []; + + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Record(string step) + { + Steps.Add(step); + if (Steps.Count >= expectedSteps) + { + Completed.TrySetResult(true); + } + } + } + + private sealed class BlockingRecorder + { + public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + private sealed class FetchJob(PipelineRecorder recorder) : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + recorder.Record("fetch"); + return Task.CompletedTask; + } + } + + private sealed class ParseJob(PipelineRecorder recorder) : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + recorder.Record("parse"); + return Task.CompletedTask; + } + } + + private sealed class MapJob(PipelineRecorder recorder) : IJob + { + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + recorder.Record("map"); + return Task.CompletedTask; + } + } + + private sealed class BlockingFetchJob(BlockingRecorder recorder) : IJob + { + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + await recorder.Release.Task.WaitAsync(cancellationToken); + recorder.Completed.TrySetResult(true); + } + } + + private sealed class TestHostApplicationLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted { get; } = CancellationToken.None; + public CancellationToken ApplicationStopping { get; } = CancellationToken.None; + public CancellationToken ApplicationStopped { get; } = CancellationToken.None; + public void StopApplication() { } + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/JobRegistrationExtensionsTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/JobRegistrationExtensionsTests.cs new file mode 100644 index 000000000..58e85fdc3 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/JobRegistrationExtensionsTests.cs @@ -0,0 +1,94 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Vndr.Cisco; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; +using StellaOps.Concelier.Connector.StellaOpsMirror; +using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.WebService.Extensions; +using Xunit; + +namespace StellaOps.Concelier.WebService.Tests; + +public sealed class JobRegistrationExtensionsTests +{ + [Fact] + public void AddBuiltInConcelierJobs_RegistersBuiltInConnectorRoutines() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["concelier:sources:stellaopsMirror:baseAddress"] = "https://mirror.stella-ops.local/api/v1" + }) + .Build(); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddOptions(); + + services.AddBuiltInConcelierJobs(configuration); + + Assert.Contains( + services, + descriptor => descriptor.ServiceType == typeof(StellaOpsMirrorConnector)); + + using var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + Assert.Equal(new Uri("https://mirror.stella-ops.local/api/v1"), options.BaseAddress); + } + + [Fact] + public void AddBuiltInConcelierJobs_RegistersMirrorJobKindsThatMatchSourceRegistryIds() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["concelier:sources:stellaopsMirror:baseAddress"] = "https://mirror.stella-ops.local/api/v1" + }) + .Build(); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddOptions(); + + services.AddBuiltInConcelierJobs(configuration); + + using var provider = services.BuildServiceProvider(); + var schedulerOptions = provider.GetRequiredService>().Value; + + Assert.Contains("source:stella-mirror:fetch", schedulerOptions.Definitions.Keys); + Assert.Contains("source:stella-mirror:parse", schedulerOptions.Definitions.Keys); + Assert.Contains("source:stella-mirror:map", schedulerOptions.Definitions.Keys); + Assert.DoesNotContain("source:stellaops-mirror:fetch", schedulerOptions.Definitions.Keys); + } + + [Fact] + public void AddBuiltInConcelierJobs_DoesNotPromoteConnectorOptionValidationToHostStartup() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddOptions(); + + services.AddBuiltInConcelierJobs(new ConfigurationBuilder().Build()); + + Assert.Contains( + services, + descriptor => descriptor.ServiceType == typeof(CiscoConnector)); + Assert.DoesNotContain( + services, + descriptor => + descriptor.ServiceType.IsGenericType && + descriptor.ServiceType.GetGenericTypeDefinition() == typeof(IConfigureOptions<>) && + string.Equals( + descriptor.ServiceType.GenericTypeArguments[0].FullName, + "Microsoft.Extensions.Options.StartupValidatorOptions", + StringComparison.Ordinal)); + + using var provider = services.BuildServiceProvider(); + var error = Assert.Throws( + () => provider.GetRequiredService>().Value); + Assert.Contains("Cisco clientId must be configured", error.Message, StringComparison.Ordinal); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/LeaseStoreWiringTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/LeaseStoreWiringTests.cs new file mode 100644 index 000000000..c1d35afac --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/LeaseStoreWiringTests.cs @@ -0,0 +1,50 @@ +using FluentAssertions; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Persistence.Postgres.Repositories; +using Xunit; + +using StellaOps.TestKit; +namespace StellaOps.Concelier.WebService.Tests; + +public sealed class LeaseStoreWiringTests +{ + [Trait("Category", TestCategories.Unit)] + [Fact] + public void Host_Resolves_PostgresLeaseStore() + { + using var factory = new LeaseStoreWebApplicationFactory(); + using var scope = factory.Services.CreateScope(); + + var store = scope.ServiceProvider.GetRequiredService(); + + store.Should().BeOfType(); + store.Should().NotBeOfType(); + } + + private sealed class LeaseStoreWebApplicationFactory : WebApplicationFactory + { + public LeaseStoreWebApplicationFactory() + { + Environment.SetEnvironmentVariable("CONCELIER__POSTGRESSTORAGE__CONNECTIONSTRING", "Host=localhost;Port=5432;Database=test-contract;Username=postgres;Password=postgres"); + Environment.SetEnvironmentVariable("CONCELIER__POSTGRESSTORAGE__COMMANDTIMEOUTSECONDS", "30"); + Environment.SetEnvironmentVariable("CONCELIER_TEST_STORAGE_DSN", "Host=localhost;Port=5432;Database=test-contract;Username=postgres;Password=postgres"); + Environment.SetEnvironmentVariable("CONCELIER_SKIP_OPTIONS_VALIDATION", "1"); + Environment.SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Testing"); + Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Testing"); + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Testing"); + builder.ConfigureServices(services => + { + services.RemoveAll(); + }); + } + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TASKS.md index e87197e20..bb91446db 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/TASKS.md @@ -9,3 +9,10 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0243-T | DONE | Revalidated 2026-01-07. | | AUDIT-0243-A | DONE | Waived (test project; revalidated 2026-01-07). | | BE8-07-TEST | DONE | Extended advisory-source endpoint coverage for total/signed/unsigned/signature-failure contract fields. | +| NOMOCK-003 | DONE | 2026-04-18: Extended `UnsupportedRuntimeWiringTests` and hardened `WebServiceEndpointsTests` mirror happy-path/auth/rate-limit proof so runtime evidence seeds mirror state through persisted stores instead of config-only domains. | +| REALPLAN-007-C | DONE | 2026-04-15: Added `LeaseStoreWiringTests` so the live WebService host resolves `ILeaseStore` to `PostgresLeaseStore`. | +| REALPLAN-007-E | DONE | 2026-04-17: Extended `UnsupportedRuntimeWiringTests` to prove affected-symbol services resolve to explicit unsupported runtime implementations and `/v1/signals/symbols/*` returns `501`. | +| REALPLAN-007-F | DOING | 2026-04-19: Updating runtime proof so the live host resolves durable affected-symbol services and the raw-ingest path can be verified against persisted observation/symbol storage. | +| NOMOCK-004 | DONE | 2026-04-18: Extended `UnsupportedRuntimeWiringTests` to prove non-testing mirror import succeeds against a real staged bundle, exposes imported mirror artifacts, and fails cleanly on checksum mismatch. | +| NOMOCK-026 | DONE | 2026-04-19: Extended mirror import tests and options tests so the live importer only accepts bundle and trust-root paths under the configured `Mirror.ImportRoot`. Verified with focused Concelier rebuilds plus direct xUnit DLL coverage after `dotnet test --filter` proved unreliable in the current test platform path. | +| REALPLAN-007-D | DONE | 2026-04-19: Extended `UnsupportedRuntimeWiringTests` so advisory-source sync compatibility routes also prove explicit `501` behavior when the live host resolves `UnsupportedJobCoordinator`. | diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/UnsupportedRuntimeWiringTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/UnsupportedRuntimeWiringTests.cs new file mode 100644 index 000000000..8c1dfd1d9 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/UnsupportedRuntimeWiringTests.cs @@ -0,0 +1,624 @@ +using FluentAssertions; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Concelier.Core.Federation; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Core.Observations; +using StellaOps.Concelier.Core.Orchestration; +using StellaOps.Concelier.Core.Raw; +using StellaOps.Concelier.Core.Signals; +using StellaOps.Replay.Core.FeedSnapshot; +using StellaOps.Concelier.WebService.Extensions; +using StellaOps.Concelier.WebService.Services; +using Moq; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Security.Claims; +using System.Text.Json; +using System.Text.Encodings.Web; +using Xunit; + +namespace StellaOps.Concelier.WebService.Tests; + +public sealed class UnsupportedRuntimeWiringTests +{ + [Fact] + public void DevelopmentHost_ResolvesUnsupportedRuntimeServices() + { + using var factory = new UnsupportedRuntimeWebApplicationFactory(); + using var scope = factory.Services.CreateScope(); + + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + scope.ServiceProvider.GetRequiredService().Should().BeOfType(); + factory.HasJobSchedulerHostedService.Should().BeFalse(); + } + + [Fact] + public async Task DevelopmentHost_JobsEndpoint_ReturnsNotImplemented() + { + using var factory = new UnsupportedRuntimeWebApplicationFactory(); + using var client = factory.CreateClient(); + + var response = await client.GetAsync("/jobs/definitions"); + var body = await response.Content.ReadAsStringAsync(); + + response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); + body.Should().Contain("NOT_IMPLEMENTED"); + body.Should().Contain("durable backend implementation"); + } + + [Fact] + public async Task DevelopmentHost_SourceSyncEndpoint_ReturnsNotImplemented() + { + using var factory = new UnsupportedRuntimeWebApplicationFactory(); + using var client = factory.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/advisory-sources/nvd/sync"); + request.Headers.Add(Program.TenantHeaderName, "tenant-a"); + + var response = await client.SendAsync(request); + var body = await response.Content.ReadAsStringAsync(); + + response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); + body.Should().Contain("NOT_IMPLEMENTED"); + body.Should().Contain("durable backend implementation"); + } + + [Fact] + public async Task DevelopmentHost_BatchSourceSyncEndpoint_ReturnsNotImplemented() + { + using var factory = new UnsupportedRuntimeWebApplicationFactory(); + using var client = factory.CreateClient(); + using var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1/advisory-sources/sync"); + request.Headers.Add(Program.TenantHeaderName, "tenant-a"); + + var response = await client.SendAsync(request); + var body = await response.Content.ReadAsStringAsync(); + + response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); + body.Should().Contain("NOT_IMPLEMENTED"); + body.Should().Contain("durable backend implementation"); + } + + [Fact] + public async Task DevelopmentHost_InternalOrchestratorEndpoint_ReturnsNotImplemented() + { + using var factory = new UnsupportedRuntimeWebApplicationFactory(); + using var client = factory.CreateClient(); + using var request = new HttpRequestMessage( + HttpMethod.Get, + $"/internal/orch/commands?connectorId=nvd&runId={Guid.NewGuid()}"); + request.Headers.Add(Program.TenantHeaderName, "tenant-a"); + + var response = await client.SendAsync(request); + var body = await response.Content.ReadAsStringAsync(); + + response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); + body.Should().Contain("NOT_IMPLEMENTED"); + body.Should().Contain("orchestrator registry"); + } + + [Fact] + public async Task DevelopmentHost_AffectedSymbolsEndpoint_ReturnsNotImplemented() + { + using var factory = new UnsupportedRuntimeWebApplicationFactory(); + using var client = factory.CreateClient(); + using var request = new HttpRequestMessage( + HttpMethod.Get, + "/v1/signals/symbols/advisory/CVE-2026-0001"); + request.Headers.Add(Program.TenantHeaderName, "tenant-a"); + + var response = await client.SendAsync(request); + var body = await response.Content.ReadAsStringAsync(); + + response.StatusCode.Should().Be(System.Net.HttpStatusCode.NotImplemented); + body.Should().Contain("NOT_IMPLEMENTED"); + body.Should().Contain("affected-symbol runtime"); + } + + [Fact] + public async Task DevelopmentHost_ConfigSeededMirrorEndpoints_IgnoreConfiguredDomains() + { + using var temp = new TemporaryDirectory(); + var exportId = "20251019T120000Z"; + var domainRoot = Path.Combine(temp.Path, exportId, "mirror", "primary"); + Directory.CreateDirectory(domainRoot); + + await File.WriteAllTextAsync( + Path.Combine(temp.Path, exportId, "mirror", "index.json"), + """{"schemaVersion":1,"domains":[{"id":"primary"}]}"""); + await File.WriteAllTextAsync( + Path.Combine(domainRoot, "manifest.json"), + """{"domainId":"primary"}"""); + + using var factory = new UnsupportedRuntimeWebApplicationFactory(new Dictionary + { + ["CONCELIER_MIRROR__ENABLED"] = "true", + ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = exportId, + ["CONCELIER_MIRROR__DOMAINS__0__ID"] = "primary", + ["CONCELIER_MIRROR__DOMAINS__0__REQUIREAUTHENTICATION"] = "false", + ["CONCELIER_MIRROR__MAXINDEXREQUESTSPERHOUR"] = "5", + }); + using var client = factory.CreateClient(); + + using var indexRequest = new HttpRequestMessage(HttpMethod.Get, "/concelier/exports/index.json"); + indexRequest.Headers.Add(Program.TenantHeaderName, "tenant-a"); + var indexResponse = await client.SendAsync(indexRequest); + + using var manifestRequest = new HttpRequestMessage(HttpMethod.Get, "/concelier/exports/mirror/primary/manifest.json"); + manifestRequest.Headers.Add(Program.TenantHeaderName, "tenant-a"); + var manifestResponse = await client.SendAsync(manifestRequest); + + indexResponse.StatusCode.Should().Be(System.Net.HttpStatusCode.NotFound); + manifestResponse.StatusCode.Should().Be(System.Net.HttpStatusCode.NotFound); + } + + [Fact] + public async Task DevelopmentHost_MirrorImportEndpoints_ImportBundleAndExposeMirrorArtifacts() + { + using var source = new TemporaryDirectory(); + using var exportRoot = new TemporaryDirectory(); + var exportId = "mirror-import-proof"; + var bundleJson = """ + { + "schemaVersion": 1, + "generatedAt": "2026-04-18T00:00:00Z", + "domainId": "primary", + "displayName": "Primary", + "exportFormat": "JSON", + "sourceIds": ["nvd"], + "advisories": [ + { + "id": "CVE-2026-0001" + } + ] + } + """; + var bundlePath = Path.Combine(source.Path, "bundle.json"); + await File.WriteAllTextAsync(bundlePath, bundleJson); + + var bundleDigest = Convert.ToHexString(SHA256.HashData(await File.ReadAllBytesAsync(bundlePath))).ToLowerInvariant(); + var manifestJson = $$""" + { + "domainId": "primary", + "displayName": "Primary", + "generatedAt": "2026-04-18T00:00:00Z", + "exports": [ + { + "key": "primary", + "exportId": "primary", + "format": "JSON", + "artifactSizeBytes": {{new FileInfo(bundlePath).Length}}, + "artifactDigest": "sha256:{{bundleDigest}}" + } + ] + } + """; + await File.WriteAllTextAsync(Path.Combine(source.Path, "manifest.json"), manifestJson); + + using var factory = new UnsupportedRuntimeWebApplicationFactory(new Dictionary + { + ["CONCELIER_MIRROR__ENABLED"] = "true", + ["CONCELIER_MIRROR__EXPORTROOT"] = exportRoot.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = source.Path, + ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = exportId, + }); + using var client = factory.CreateClient(); + + using var importRequest = new HttpRequestMessage(HttpMethod.Post, "/api/v1/advisory-sources/mirror/import") + { + Content = JsonContent.Create(new { bundlePath = source.Path }) + }; + importRequest.Headers.Add(Program.TenantHeaderName, "tenant-a"); + + var importResponse = await client.SendAsync(importRequest); + var importBody = await importResponse.Content.ReadAsStringAsync(); + + using var statusRequest = new HttpRequestMessage(HttpMethod.Get, "/api/v1/advisory-sources/mirror/import/status"); + statusRequest.Headers.Add(Program.TenantHeaderName, "tenant-a"); + + var statusResponse = await client.SendAsync(statusRequest); + var statusBody = await statusResponse.Content.ReadAsStringAsync(); + + using var manifestRequest = new HttpRequestMessage(HttpMethod.Get, "/concelier/exports/mirror/primary/manifest.json"); + manifestRequest.Headers.Add(Program.TenantHeaderName, "tenant-a"); + var manifestResponse = await client.SendAsync(manifestRequest); + var manifestBody = await manifestResponse.Content.ReadAsStringAsync(); + + importResponse.StatusCode.Should().Be(System.Net.HttpStatusCode.OK, importBody); + importBody.Should().Contain("completed"); + statusResponse.StatusCode.Should().Be(System.Net.HttpStatusCode.OK, statusBody); + statusBody.Should().Contain("\"hasImport\":true"); + statusBody.Should().Contain("\"status\":\"completed\""); + statusBody.Should().Contain("\"domainId\":\"primary\""); + statusBody.Should().Contain("Detached JWS signature was not found"); + manifestResponse.StatusCode.Should().Be(System.Net.HttpStatusCode.OK, manifestBody); + + using var manifestDocument = JsonDocument.Parse(manifestBody); + manifestDocument.RootElement.GetProperty("domainId").GetString().Should().Be("primary"); + } + + [Fact] + public async Task DevelopmentHost_MirrorImportEndpoints_RejectChecksumMismatch() + { + using var source = new TemporaryDirectory(); + using var exportRoot = new TemporaryDirectory(); + + var bundlePath = Path.Combine(source.Path, "bundle.json"); + await File.WriteAllTextAsync( + bundlePath, + """ + { + "schemaVersion": 1, + "generatedAt": "2026-04-18T00:00:00Z", + "domainId": "primary", + "displayName": "Primary", + "exportFormat": "JSON", + "sourceIds": ["nvd"], + "advisories": [] + } + """); + var bundleLength = new FileInfo(bundlePath).Length; + await File.WriteAllTextAsync( + Path.Combine(source.Path, "manifest.json"), + $$""" + { + "domainId": "primary", + "displayName": "Primary", + "generatedAt": "2026-04-18T00:00:00Z", + "exports": [ + { + "key": "primary", + "exportId": "primary", + "format": "JSON", + "artifactSizeBytes": {{bundleLength}}, + "artifactDigest": "sha256:deadbeef" + } + ] + } + """); + + using var factory = new UnsupportedRuntimeWebApplicationFactory(new Dictionary + { + ["CONCELIER_MIRROR__ENABLED"] = "true", + ["CONCELIER_MIRROR__EXPORTROOT"] = exportRoot.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = source.Path, + ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = "mirror-import-failure", + }); + using var client = factory.CreateClient(); + + using var importRequest = new HttpRequestMessage(HttpMethod.Post, "/api/v1/advisory-sources/mirror/import") + { + Content = JsonContent.Create(new { bundlePath = source.Path }) + }; + importRequest.Headers.Add(Program.TenantHeaderName, "tenant-a"); + + var importResponse = await client.SendAsync(importRequest); + var importBody = await importResponse.Content.ReadAsStringAsync(); + + using var statusRequest = new HttpRequestMessage(HttpMethod.Get, "/api/v1/advisory-sources/mirror/import/status"); + statusRequest.Headers.Add(Program.TenantHeaderName, "tenant-a"); + var statusResponse = await client.SendAsync(statusRequest); + var statusBody = await statusResponse.Content.ReadAsStringAsync(); + + importResponse.StatusCode.Should().Be(System.Net.HttpStatusCode.BadRequest, importBody); + importBody.Should().Contain("\"status\":\"failed\""); + importBody.Should().Contain("digest mismatch"); + statusResponse.StatusCode.Should().Be(System.Net.HttpStatusCode.OK, statusBody); + statusBody.Should().Contain("\"status\":\"failed\""); + statusBody.Should().Contain("digest mismatch"); + } + + [Fact] + public async Task DevelopmentHost_MirrorImportEndpoints_RejectPathsOutsideAllowlistedImportRoot() + { + using var source = new TemporaryDirectory(); + using var importRoot = new TemporaryDirectory(); + using var exportRoot = new TemporaryDirectory(); + + await File.WriteAllTextAsync( + Path.Combine(source.Path, "bundle.json"), + """ + { + "schemaVersion": 1, + "generatedAt": "2026-04-18T00:00:00Z", + "domainId": "primary", + "displayName": "Primary", + "exportFormat": "JSON", + "sourceIds": ["nvd"], + "advisories": [] + } + """); + await File.WriteAllTextAsync( + Path.Combine(source.Path, "manifest.json"), + """ + { + "domainId": "primary", + "displayName": "Primary", + "generatedAt": "2026-04-18T00:00:00Z", + "exports": [ + { + "key": "primary", + "exportId": "primary", + "format": "JSON", + "artifactSizeBytes": 120, + "artifactDigest": "sha256:deadbeef" + } + ] + } + """); + + using var factory = new UnsupportedRuntimeWebApplicationFactory(new Dictionary + { + ["CONCELIER_MIRROR__ENABLED"] = "true", + ["CONCELIER_MIRROR__EXPORTROOT"] = exportRoot.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = importRoot.Path, + ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = "mirror-import-allowlist", + }); + using var client = factory.CreateClient(); + + using var importRequest = new HttpRequestMessage(HttpMethod.Post, "/api/v1/advisory-sources/mirror/import") + { + Content = JsonContent.Create(new { bundlePath = source.Path }) + }; + importRequest.Headers.Add(Program.TenantHeaderName, "tenant-a"); + + var importResponse = await client.SendAsync(importRequest); + var importBody = await importResponse.Content.ReadAsStringAsync(); + + importResponse.StatusCode.Should().Be(System.Net.HttpStatusCode.BadRequest, importBody); + importBody.Should().Contain("\"status\":\"failed\""); + importBody.Should().Contain("must stay within the configured mirror import root"); + } + + private sealed class UnsupportedRuntimeWebApplicationFactory : WebApplicationFactory + { + private readonly Dictionary _previousEnvironment = new(StringComparer.OrdinalIgnoreCase); + + public bool HasJobSchedulerHostedService { get; private set; } + + public UnsupportedRuntimeWebApplicationFactory(IReadOnlyDictionary? environmentOverrides = null) + { + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__CONNECTIONSTRING", "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres"); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__ENABLED", "true"); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__COMMANDTIMEOUTSECONDS", "30"); + SetEnvironmentVariable("CONCELIER_POSTGRESSTORAGE__SCHEMANAME", "vuln"); + SetEnvironmentVariable("CONCELIER_POSTGRES_DSN", "Host=localhost;Port=5432;Database=concelier_test;Username=postgres;Password=postgres"); + SetEnvironmentVariable("CONCELIER_SKIP_OPTIONS_VALIDATION", "1"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING", "false"); + SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLEMETRICS", "false"); + SetEnvironmentVariable("CONCELIER_AUTHORITY__ENABLED", "false"); + SetEnvironmentVariable("CONCELIER_AUTHORITY__REQUIREDSCOPES__0", "concelier.jobs.trigger"); + SetEnvironmentVariable("DOTNET_ENVIRONMENT", "Production"); + SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Production"); + + if (environmentOverrides is not null) + { + foreach (var entry in environmentOverrides) + { + SetEnvironmentVariable(entry.Key, entry.Value); + } + } + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Production"); + builder.ConfigureServices(services => + { + HasJobSchedulerHostedService = services.Any(descriptor => + descriptor.ServiceType == typeof(IHostedService) && + descriptor.ImplementationType == typeof(JobSchedulerHostedService)); + + services.AddHttpContextAccessor(); + services.AddAuthentication("Test") + .AddScheme("Test", _ => { }) + .AddScheme("StellaOpsBearer", _ => { }); + services.AddAuthorization(options => + { + options.AddPolicy("Concelier.Sources.Manage", policy => policy.RequireAssertion(_ => true)); + options.AddPolicy("Concelier.Advisories.Read", policy => policy.RequireAssertion(_ => true)); + }); + services.TryAddSingleton>(_ => () => Guid.Parse("11111111-1111-1111-1111-111111111111")); + services.RemoveAll(); + services.RemoveAll(); + services.RemoveAll(); + services.TryAddSingleton(Mock.Of()); + services.TryAddSingleton(Mock.Of()); + services.TryAddSingleton(Mock.Of()); + services.TryAddSingleton(CreateFeedSnapshotCoordinator()); + services.RemoveAll(); + services.AddSingleton(); + services.RemoveAll(); + services.AddSingleton(); + services.RemoveAll(); + }); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + foreach (var entry in _previousEnvironment) + { + Environment.SetEnvironmentVariable(entry.Key, entry.Value); + } + } + + base.Dispose(disposing); + } + + private void SetEnvironmentVariable(string name, string? value) + { + _previousEnvironment[name] = Environment.GetEnvironmentVariable(name); + Environment.SetEnvironmentVariable(name, value); + } + + private static IFeedSnapshotCoordinator CreateFeedSnapshotCoordinator() + { + var mock = new Mock(); + mock.SetupGet(x => x.RegisteredSources).Returns(Array.Empty()); + return mock.Object; + } + } + + private sealed class InMemoryMirrorDomainStore : IMirrorDomainStore + { + private readonly object _sync = new(); + private readonly Dictionary _domains = new(StringComparer.OrdinalIgnoreCase); + + public IReadOnlyList GetAllDomains() + { + lock (_sync) + { + return _domains.Values.Select(Clone).ToArray(); + } + } + + public MirrorDomainRecord? GetDomain(string domainId) + { + lock (_sync) + { + return _domains.TryGetValue(domainId, out var domain) ? Clone(domain) : null; + } + } + + public Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + lock (_sync) + { + _domains[domain.Id] = Clone(domain); + } + + return Task.CompletedTask; + } + + public Task DeleteDomainAsync(string domainId, CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + lock (_sync) + { + _domains.Remove(domainId); + } + + return Task.CompletedTask; + } + + private static MirrorDomainRecord Clone(MirrorDomainRecord domain) => new() + { + Id = domain.Id, + DisplayName = domain.DisplayName, + SourceIds = [.. domain.SourceIds], + ExportFormat = domain.ExportFormat, + RequireAuthentication = domain.RequireAuthentication, + MaxIndexRequestsPerHour = domain.MaxIndexRequestsPerHour, + MaxDownloadRequestsPerHour = domain.MaxDownloadRequestsPerHour, + SigningEnabled = domain.SigningEnabled, + SigningAlgorithm = domain.SigningAlgorithm, + SigningKeyId = domain.SigningKeyId, + Exports = domain.Exports + .Select(export => new MirrorExportRecord + { + Key = export.Key, + Format = export.Format, + Filters = new Dictionary(export.Filters, StringComparer.Ordinal), + }) + .ToList(), + CreatedAt = domain.CreatedAt, + UpdatedAt = domain.UpdatedAt, + LastGeneratedAt = domain.LastGeneratedAt, + LastGenerateTriggeredAt = domain.LastGenerateTriggeredAt, + BundleSizeBytes = domain.BundleSizeBytes, + AdvisoryCount = domain.AdvisoryCount, + }; + } + + private sealed class InMemoryMirrorBundleImportStore : IMirrorBundleImportStore + { + private MirrorImportStatusRecord? _status; + + public MirrorImportStatusRecord? GetLatestStatus() => _status; + + public void SetStatus(MirrorImportStatusRecord status) => _status = status; + } + + private sealed class TemporaryDirectory : IDisposable + { + public TemporaryDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "concelier-mirror-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(Path); + } + + public string Path { get; } + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + } + + private sealed class AlwaysAllowAuthHandler : AuthenticationHandler + { + public AlwaysAllowAuthHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder) + : base(options, logger, encoder) + { + } + + protected override Task HandleAuthenticateAsync() + { + var scopes = new[] + { + StellaOpsScopes.ConcelierJobsTrigger, + StellaOpsScopes.AdvisoryRead, + StellaOpsScopes.IntegrationWrite, + StellaOpsScopes.IntegrationOperate, + }; + + var identity = new ClaimsIdentity( + [ + new Claim(ClaimTypes.NameIdentifier, "unsupported-runtime"), + new Claim(ClaimTypes.Name, "unsupported-runtime"), + new Claim(StellaOpsClaimTypes.Tenant, "tenant-a"), + new Claim(StellaOpsClaimTypes.Scope, string.Join(' ', scopes)), + new Claim(StellaOpsClaimTypes.ScopeItem, StellaOpsScopes.ConcelierJobsTrigger), + new Claim(StellaOpsClaimTypes.ScopeItem, StellaOpsScopes.AdvisoryRead), + new Claim(StellaOpsClaimTypes.ScopeItem, StellaOpsScopes.IntegrationWrite), + new Claim(StellaOpsClaimTypes.ScopeItem, StellaOpsScopes.IntegrationOperate), + ], Scheme.Name); + + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(ticket)); + } + } +} diff --git a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs index 78af8a5ea..3ea7a28d6 100644 --- a/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs +++ b/src/Concelier/__Tests/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs @@ -24,6 +24,7 @@ using Microsoft.IdentityModel.Logging; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StellaOps.Concelier.InMemoryRunner; @@ -45,8 +46,11 @@ using StellaOps.Concelier.Core.Observations; using StellaOps.Concelier.Core.Linksets; using StellaOps.Concelier.Models.Observations; using StellaOps.Concelier.Persistence.Postgres; +using StellaOps.Concelier.Persistence.Postgres.Models; +using StellaOps.Concelier.Persistence.Postgres.Repositories; using StellaOps.Concelier.RawModels; using StellaOps.Concelier.WebService.Jobs; +using StellaOps.Concelier.WebService.Extensions; using StellaOps.Concelier.WebService.Options; using StellaOps.Concelier.WebService.Contracts; using StellaOps.Concelier.WebService; @@ -59,6 +63,7 @@ using Microsoft.IdentityModel.Protocols.OpenIdConnect; using StellaOps.Concelier.WebService.Diagnostics; using Microsoft.IdentityModel.Tokens; using StellaOps.Cryptography; +using StellaOps.Concelier.WebService.Tests.Fixtures; using DsseProvenance = StellaOps.Provenance.DsseProvenance; using TrustInfo = StellaOps.Provenance.TrustInfo; using DocumentObject = StellaOps.Concelier.Documents.DocumentObject; @@ -1406,17 +1411,15 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime { ["CONCELIER_MIRROR__ENABLED"] = "true", ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = temp.Path, ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = exportId, - ["CONCELIER_MIRROR__DOMAINS__0__ID"] = "primary", - ["CONCELIER_MIRROR__DOMAINS__0__DISPLAYNAME"] = "Primary", - ["CONCELIER_MIRROR__DOMAINS__0__REQUIREAUTHENTICATION"] = "false", - ["CONCELIER_MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR"] = "5", ["CONCELIER_MIRROR__MAXINDEXREQUESTSPERHOUR"] = "5" }; using var factory = new ConcelierApplicationFactory(_runner.ConnectionString, environmentOverrides: environment); using var client = factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test-tenant"); + await CreateMirrorDomainAsync(client, "primary", requireAuthentication: false, maxDownloadRequestsPerHour: 5); var indexResponse = await client.GetAsync("/concelier/exports/index.json"); Assert.Equal(HttpStatusCode.OK, indexResponse.StatusCode); @@ -1453,10 +1456,8 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime { ["CONCELIER_MIRROR__ENABLED"] = "true", ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = temp.Path, ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = exportId, - ["CONCELIER_MIRROR__DOMAINS__0__ID"] = "secure", - ["CONCELIER_MIRROR__DOMAINS__0__REQUIREAUTHENTICATION"] = "true", - ["CONCELIER_MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR"] = "5", ["CONCELIER_AUTHORITY__ENABLED"] = "true", ["CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK"] = "false", ["CONCELIER_AUTHORITY__ISSUER"] = "https://authority.example", @@ -1487,6 +1488,37 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime using var client = factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test-tenant"); + using (var scope = factory.Services.CreateScope()) + { + var domainStore = scope.ServiceProvider.GetRequiredService(); + await domainStore.SaveDomainAsync( + new MirrorDomainRecord + { + Id = "secure", + DisplayName = "secure", + SourceIds = ["nvd"], + ExportFormat = "JSON", + RequireAuthentication = true, + MaxIndexRequestsPerHour = 60, + MaxDownloadRequestsPerHour = 5, + SigningEnabled = false, + SigningAlgorithm = "HMAC-SHA256", + SigningKeyId = string.Empty, + Exports = + [ + new MirrorExportRecord + { + Key = "secure", + Format = "JSON", + Filters = new Dictionary(StringComparer.Ordinal), + } + ], + CreatedAt = DateTimeOffset.UtcNow, + UpdatedAt = DateTimeOffset.UtcNow, + }, + CancellationToken.None); + } + var response = await client.GetAsync("/concelier/exports/mirror/secure/manifest.json"); Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); var authHeader = Assert.Single(response.Headers.WwwAuthenticate); @@ -1511,16 +1543,16 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime { ["CONCELIER_MIRROR__ENABLED"] = "true", ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = temp.Path, ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = exportId, ["CONCELIER_MIRROR__MAXINDEXREQUESTSPERHOUR"] = "1", - ["CONCELIER_MIRROR__DOMAINS__0__ID"] = "primary", - ["CONCELIER_MIRROR__DOMAINS__0__REQUIREAUTHENTICATION"] = "false", - ["CONCELIER_MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR"] = "1" + ["CONCELIER_MIRROR__MAXDOWNLOADREQUESTSPERHOUR"] = "1" }; using var factory = new ConcelierApplicationFactory(_runner.ConnectionString, environmentOverrides: environment); using var client = factory.CreateClient(); client.DefaultRequestHeaders.Add("X-Stella-Tenant", "test-tenant"); + await CreateMirrorDomainAsync(client, "primary", requireAuthentication: false, maxDownloadRequestsPerHour: 1); var okResponse = await client.GetAsync("/concelier/exports/index.json"); Assert.Equal(HttpStatusCode.OK, okResponse.StatusCode); @@ -1540,6 +1572,7 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime { ["CONCELIER_MIRROR__ENABLED"] = "true", ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__IMPORTROOT"] = temp.Path, ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = "latest", ["CONCELIER_MIRROR__MAXINDEXREQUESTSPERHOUR"] = "5", ["CONCELIER_MIRROR__MAXDOWNLOADREQUESTSPERHOUR"] = "5" @@ -1591,11 +1624,6 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime var endpointsResponse = await client.GetAsync("/api/v1/advisory-sources/mirror/domains/primary/endpoints"); Assert.Equal(HttpStatusCode.OK, endpointsResponse.StatusCode); - var generateResponse = await client.PostAsync("/api/v1/advisory-sources/mirror/domains/primary/generate", content: null); - Assert.Equal(HttpStatusCode.OK, generateResponse.StatusCode); - Assert.True(File.Exists(Path.Combine(temp.Path, "latest", "mirror", "primary", "bundle.json"))); - Assert.True(File.Exists(Path.Combine(temp.Path, "latest", "mirror", "primary", "manifest.json"))); - var healthResponse = await client.GetAsync("/api/v1/advisory-sources/mirror/health"); Assert.Equal(HttpStatusCode.OK, healthResponse.StatusCode); var healthJson = JsonDocument.Parse(await healthResponse.Content.ReadAsStringAsync()); @@ -1632,6 +1660,40 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime Assert.True(importStatusJson.RootElement.GetProperty("hasImport").GetBoolean()); } + private static async Task CreateMirrorDomainAsync( + HttpClient client, + string domainId, + bool requireAuthentication, + int maxDownloadRequestsPerHour) + { + var createResponse = await client.PostAsJsonAsync( + "/api/v1/advisory-sources/mirror/domains", + new + { + domainId, + displayName = domainId, + sourceIds = new[] { "nvd" }, + exportFormat = "JSON", + rateLimits = new + { + indexRequestsPerHour = 60, + downloadRequestsPerHour = maxDownloadRequestsPerHour + }, + requireAuthentication, + signing = new + { + enabled = false, + algorithm = "HMAC-SHA256", + keyId = string.Empty + } + }); + + var body = await createResponse.Content.ReadAsStringAsync(); + Assert.True( + createResponse.StatusCode == HttpStatusCode.Created, + $"Mirror domain creation failed with {(int)createResponse.StatusCode}: {body}"); + } + [Fact] public void MergeModuleDisabledByDefault() { @@ -2324,6 +2386,60 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime builder.ConfigureTestServices(services => { + var startupMigrationHostedServices = services + .Where(descriptor => + descriptor.ServiceType == typeof(IHostedService) && + descriptor.ImplementationFactory?.Method.DeclaringType?.FullName?.Contains( + "MigrationServiceExtensions", + StringComparison.Ordinal) == true) + .ToArray(); + foreach (var descriptor in startupMigrationHostedServices) + { + services.Remove(descriptor); + } + + if (_authorityConfigure is null) + { + services.AddAuthentication(options => + { + options.DefaultAuthenticateScheme = ConcelierTestAuthHandler.SchemeName; + options.DefaultChallengeScheme = ConcelierTestAuthHandler.SchemeName; + }) + .AddScheme( + ConcelierTestAuthHandler.SchemeName, static _ => { }); + + services.AddAuthorization(options => + { + foreach (var policy in new[] + { + "Concelier.Jobs.Trigger", + "Concelier.Observations.Read", + "Concelier.Advisories.Ingest", + "Concelier.Advisories.Read", + "Concelier.Aoc.Verify", + "Concelier.Canonical.Read", + "Concelier.Canonical.Ingest", + "Concelier.Interest.Read", + "Concelier.Interest.Admin", + "Concelier.Sources.Manage", + }) + { + options.AddPolicy(policy, builder => builder.RequireAssertion(static _ => true)); + } + }); + } + + services.RemoveAll(); + services.AddSingleton(); + services.RemoveAll(); + services.AddSingleton(); + services.RemoveAll(); + services.AddSingleton(); + services.RemoveAll(); + services.AddSingleton(); + services.RemoveAll(); + services.AddSingleton(); + services.AddSingleton(); // Ensure JWT handler doesn't map claims to different types @@ -3586,6 +3702,252 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime return JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web)); } + private sealed class InMemoryMirrorDomainStore : IMirrorDomainStore + { + private readonly object syncRoot = new(); + private readonly Dictionary domains = new(StringComparer.OrdinalIgnoreCase); + + public IReadOnlyList GetAllDomains() + { + lock (syncRoot) + { + return domains.Values.Select(Clone).ToArray(); + } + } + + public MirrorDomainRecord? GetDomain(string domainId) + { + lock (syncRoot) + { + return domains.TryGetValue(domainId, out var domain) ? Clone(domain) : null; + } + } + + public Task SaveDomainAsync(MirrorDomainRecord domain, CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + lock (syncRoot) + { + domains[domain.Id] = Clone(domain); + } + + return Task.CompletedTask; + } + + public Task DeleteDomainAsync(string domainId, CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + lock (syncRoot) + { + domains.Remove(domainId); + } + + return Task.CompletedTask; + } + + private static MirrorDomainRecord Clone(MirrorDomainRecord domain) => new() + { + Id = domain.Id, + DisplayName = domain.DisplayName, + SourceIds = [.. domain.SourceIds], + ExportFormat = domain.ExportFormat, + RequireAuthentication = domain.RequireAuthentication, + MaxIndexRequestsPerHour = domain.MaxIndexRequestsPerHour, + MaxDownloadRequestsPerHour = domain.MaxDownloadRequestsPerHour, + SigningEnabled = domain.SigningEnabled, + SigningAlgorithm = domain.SigningAlgorithm, + SigningKeyId = domain.SigningKeyId, + Exports = domain.Exports + .Select(export => new MirrorExportRecord + { + Key = export.Key, + Format = export.Format, + Filters = new Dictionary(export.Filters, StringComparer.Ordinal), + }) + .ToList(), + CreatedAt = domain.CreatedAt, + UpdatedAt = domain.UpdatedAt, + LastGeneratedAt = domain.LastGeneratedAt, + LastGenerateTriggeredAt = domain.LastGenerateTriggeredAt, + BundleSizeBytes = domain.BundleSizeBytes, + AdvisoryCount = domain.AdvisoryCount, + }; + } + + private sealed class InMemoryMirrorConfigStore : IMirrorConfigStore + { + private readonly object syncRoot = new(); + private MirrorConfigRecord config = new(); + + public MirrorConfigRecord GetConfig() + { + lock (syncRoot) + { + return new MirrorConfigRecord + { + Mode = config.Mode, + OutputRoot = config.OutputRoot, + ConsumerBaseAddress = config.ConsumerBaseAddress, + SigningEnabled = config.SigningEnabled, + SigningAlgorithm = config.SigningAlgorithm, + SigningKeyId = config.SigningKeyId, + AutoRefreshEnabled = config.AutoRefreshEnabled, + RefreshIntervalMinutes = config.RefreshIntervalMinutes, + }; + } + } + + public Task UpdateConfigAsync(UpdateMirrorConfigRequest request, CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + lock (syncRoot) + { + config = new MirrorConfigRecord + { + Mode = request.Mode ?? config.Mode, + OutputRoot = config.OutputRoot, + ConsumerBaseAddress = request.ConsumerBaseAddress ?? config.ConsumerBaseAddress, + SigningEnabled = request.Signing?.Enabled ?? config.SigningEnabled, + SigningAlgorithm = request.Signing?.Algorithm ?? config.SigningAlgorithm, + SigningKeyId = request.Signing?.KeyId ?? config.SigningKeyId, + AutoRefreshEnabled = request.AutoRefreshEnabled ?? config.AutoRefreshEnabled, + RefreshIntervalMinutes = request.RefreshIntervalMinutes ?? config.RefreshIntervalMinutes, + }; + } + + return Task.CompletedTask; + } + } + + private sealed class InMemoryMirrorConsumerConfigStore : IMirrorConsumerConfigStore + { + private readonly object syncRoot = new(); + private ConsumerConfigResponse config = new() + { + Connected = false, + }; + + public ConsumerConfigResponse GetConsumerConfig() + { + lock (syncRoot) + { + return config with + { + Signature = config.Signature is null + ? null + : new ConsumerSignatureResponse + { + Enabled = config.Signature.Enabled, + Algorithm = config.Signature.Algorithm, + KeyId = config.Signature.KeyId, + PublicKeyPem = config.Signature.PublicKeyPem, + } + }; + } + } + + public Task SetConsumerConfigAsync(ConsumerConfigRequest request, CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + lock (syncRoot) + { + config = new ConsumerConfigResponse + { + BaseAddress = request.BaseAddress, + DomainId = request.DomainId, + IndexPath = string.IsNullOrWhiteSpace(request.IndexPath) ? "/concelier/exports/index.json" : request.IndexPath, + HttpTimeoutSeconds = request.HttpTimeoutSeconds ?? 30, + Signature = request.Signature is null + ? null + : new ConsumerSignatureResponse + { + Enabled = request.Signature.Enabled, + Algorithm = request.Signature.Algorithm, + KeyId = request.Signature.KeyId, + PublicKeyPem = request.Signature.PublicKeyPem, + }, + Connected = true, + LastSync = DateTimeOffset.UtcNow, + }; + } + + return Task.CompletedTask; + } + } + + private sealed class StubSourceRepository : ISourceRepository + { + private readonly List sources = + [ + new SourceEntity + { + Id = Guid.Parse("0e94e643-4045-4af8-b0c4-f4f04f769279"), + Key = "nvd", + Name = "NVD", + SourceType = "nvd", + Url = "https://nvd.nist.gov", + Priority = 100, + Enabled = true, + Config = """{"syncIntervalMinutes":360,"localPath":"/data/mirrors/nvd"}""", + CreatedAt = DateTimeOffset.Parse("2026-02-18T00:00:00Z", CultureInfo.InvariantCulture), + UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z", CultureInfo.InvariantCulture), + }, + new SourceEntity + { + Id = Guid.Parse("db66a4a1-b0bf-4bb4-a9cb-b9ea2e057613"), + Key = "osv", + Name = "OSV", + SourceType = "osv", + Url = "https://osv.dev", + Priority = 90, + Enabled = true, + Config = """{"syncIntervalMinutes":360,"localPath":"/data/mirrors/osv"}""", + CreatedAt = DateTimeOffset.Parse("2026-02-18T00:00:00Z", CultureInfo.InvariantCulture), + UpdatedAt = DateTimeOffset.Parse("2026-02-19T00:00:00Z", CultureInfo.InvariantCulture), + } + ]; + + public Task UpsertAsync(SourceEntity source, CancellationToken cancellationToken = default) + { + var index = sources.FindIndex(existing => + existing.Id == source.Id || + string.Equals(existing.Key, source.Key, StringComparison.OrdinalIgnoreCase)); + if (index >= 0) + { + sources[index] = source; + } + else + { + sources.Add(source); + } + + return Task.FromResult(source); + } + + public Task GetByIdAsync(Guid id, CancellationToken cancellationToken = default) + => Task.FromResult(sources.FirstOrDefault(source => source.Id == id)); + + public Task GetByKeyAsync(string key, CancellationToken cancellationToken = default) + => Task.FromResult(sources.FirstOrDefault(source => string.Equals(source.Key, key, StringComparison.OrdinalIgnoreCase))); + + public Task> ListAsync(bool? enabled = null, CancellationToken cancellationToken = default) + { + var items = sources + .Where(source => enabled is null || source.Enabled == enabled.Value) + .ToList(); + return Task.FromResult>(items); + } + } + + private sealed class InMemoryMirrorBundleImportStore : IMirrorBundleImportStore + { + private MirrorImportStatusRecord? status; + + public MirrorImportStatusRecord? GetLatestStatus() => status; + + public void SetStatus(MirrorImportStatusRecord status) => this.status = status; + } + private sealed class TempDirectory : IDisposable { public string Path { get; } diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/ExcititorMigrationTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/ExcititorMigrationTests.cs index a513e9628..27c7ccbab 100644 --- a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/ExcititorMigrationTests.cs +++ b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/ExcititorMigrationTests.cs @@ -5,10 +5,11 @@ // Description: Model S1 migration tests for Excititor.Storage // ----------------------------------------------------------------------------- -using System.Reflection; using Dapper; using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; using Npgsql; +using StellaOps.Infrastructure.Postgres.Migrations; using StellaOps.Excititor.Persistence.Postgres; using StellaOps.TestKit; using Testcontainers.PostgreSql; @@ -59,15 +60,20 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime await using var connection = new NpgsqlConnection(connectionString); await connection.OpenAsync(); - var tables = await connection.QueryAsync( + var vexTables = (await connection.QueryAsync( @"SELECT table_name FROM information_schema.tables - WHERE table_schema = 'public' - ORDER BY table_name"); + WHERE table_schema = 'vex' + ORDER BY table_name")).ToList(); - var tableList = tables.ToList(); + var excititorTables = (await connection.QueryAsync( + @"SELECT table_name FROM information_schema.tables + WHERE table_schema = 'excititor' + ORDER BY table_name")).ToList(); - // Verify migration tracking table exists - tableList.Should().Contain("__migrations", "Migration tracking table should exist"); + vexTables.Should().Contain("linksets", "core VEX linkset storage must exist"); + vexTables.Should().Contain("vex_raw_documents", "raw VEX document storage must exist"); + vexTables.Should().Contain("schema_migrations", "authoritative migration tracking must exist in the module schema"); + excititorTables.Should().Contain("source_trust_vectors", "Excititor calibration storage must exist"); } [Fact] @@ -82,7 +88,7 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime await connection.OpenAsync(); var migrationsApplied = await connection.QueryAsync( - "SELECT migration_id FROM __migrations ORDER BY applied_at"); + "SELECT migration_name FROM vex.schema_migrations ORDER BY applied_at"); var migrationList = migrationsApplied.ToList(); migrationList.Should().NotBeEmpty("migrations should be tracked"); @@ -107,11 +113,11 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime await connection.OpenAsync(); var migrationCount = await connection.ExecuteScalarAsync( - "SELECT COUNT(*) FROM __migrations"); + "SELECT COUNT(*) FROM vex.schema_migrations"); // Count unique migrations var uniqueMigrations = await connection.ExecuteScalarAsync( - "SELECT COUNT(DISTINCT migration_id) FROM __migrations"); + "SELECT COUNT(DISTINCT migration_name) FROM vex.schema_migrations"); migrationCount.Should().Be(uniqueMigrations, "each migration should only be recorded once"); @@ -124,17 +130,29 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime var connectionString = _container.GetConnectionString(); await ApplyAllMigrationsAsync(connectionString); - // Assert - Verify indexes exist + // Assert - Verify core persisted tables have structural constraints. await using var connection = new NpgsqlConnection(connectionString); await connection.OpenAsync(); - var indexes = await connection.QueryAsync( - @"SELECT indexname FROM pg_indexes - WHERE schemaname = 'public' - ORDER BY indexname"); + var constraints = (await connection.QueryAsync<(string TableName, string ConstraintType)>( + @"SELECT table_name, constraint_type + FROM information_schema.table_constraints + WHERE table_schema = 'vex' + AND table_name IN ('linksets', 'claims', 'graph_overlays', 'attestations') + ORDER BY table_name, constraint_type")).ToList(); - var indexList = indexes.ToList(); - indexList.Should().NotBeNull("indexes collection should exist"); + constraints.Should().Contain( + item => item.TableName == "linksets" && item.ConstraintType == "PRIMARY KEY", + "linksets must be keyed for deterministic replay"); + constraints.Should().Contain( + item => item.TableName == "claims" && item.ConstraintType == "PRIMARY KEY", + "claims must be durably keyed"); + constraints.Should().Contain( + item => item.TableName == "graph_overlays" && item.ConstraintType == "PRIMARY KEY", + "graph overlays must be durably keyed"); + constraints.Should().Contain( + item => item.TableName == "attestations" && item.ConstraintType == "PRIMARY KEY", + "attestations must be durably keyed"); } [Fact] @@ -143,51 +161,17 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime // Arrange var connectionString = _container.GetConnectionString(); - // Act - Apply migrations in sequence - var migrationFiles = GetMigrationFiles(); + // Act - Apply migrations through the shared runtime runner. + await ApplyAllMigrationsAsync(connectionString); await using var connection = new NpgsqlConnection(connectionString); await connection.OpenAsync(); - // Create migration tracking table first - await connection.ExecuteAsync(@" - CREATE TABLE IF NOT EXISTS __migrations ( - id SERIAL PRIMARY KEY, - migration_id TEXT NOT NULL UNIQUE, - applied_at TIMESTAMPTZ DEFAULT NOW() - )"); - - // Apply each migration in order - int appliedCount = 0; - foreach (var migrationFile in migrationFiles.OrderBy(f => f)) - { - var migrationId = Path.GetFileName(migrationFile); - - // Check if already applied - var alreadyApplied = await connection.ExecuteScalarAsync( - "SELECT COUNT(*) FROM __migrations WHERE migration_id = @Id", - new { Id = migrationId }); - - if (alreadyApplied > 0) - continue; - - // Apply migration - var sql = GetMigrationContent(migrationFile); - if (!string.IsNullOrWhiteSpace(sql)) - { - await connection.ExecuteAsync(sql); - await connection.ExecuteAsync( - "INSERT INTO __migrations (migration_id) VALUES (@Id)", - new { Id = migrationId }); - appliedCount++; - } - } - - // Assert - Migrations should be applied (if any exist) + // Assert - startup migrations should be tracked by the shared ledger. var totalMigrations = await connection.ExecuteScalarAsync( - "SELECT COUNT(*) FROM __migrations"); + "SELECT COUNT(*) FROM vex.schema_migrations"); - totalMigrations.Should().BeGreaterThanOrEqualTo(0); + totalMigrations.Should().BeGreaterThan(0); } [Fact] @@ -205,12 +189,11 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime @"SELECT tc.constraint_name FROM information_schema.table_constraints tc WHERE tc.constraint_type = 'FOREIGN KEY' - AND tc.table_schema = 'public' + AND tc.table_schema = 'vex' ORDER BY tc.constraint_name"); var fkList = foreignKeys.ToList(); - // Foreign keys may or may not exist depending on schema design - fkList.Should().NotBeNull(); + fkList.Should().NotBeEmpty("core VEX tables reference each other through foreign keys"); } [Fact] @@ -226,13 +209,20 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime var tables = await connection.QueryAsync( @"SELECT table_name FROM information_schema.tables - WHERE table_schema = 'public' - AND table_name LIKE '%vex%' OR table_name LIKE '%linkset%' + WHERE table_schema = 'vex' + AND ( + table_name LIKE '%vex%' + OR table_name LIKE '%linkset%' + OR table_name IN ('claims', 'graph_overlays', 'attestations', 'schema_migrations') + ) ORDER BY table_name"); var tableList = tables.ToList(); - // VEX tables may or may not exist depending on migration state - tableList.Should().NotBeNull(); + tableList.Should().Contain("linksets"); + tableList.Should().Contain("vex_raw_documents"); + tableList.Should().Contain("claims"); + tableList.Should().Contain("graph_overlays"); + tableList.Should().Contain("attestations"); } [Fact] @@ -250,6 +240,18 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime WHERE table_schema = 'vex' AND table_name = 'claims'"); + var graphOverlaysTableExists = await connection.ExecuteScalarAsync( + @"SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema = 'vex' + AND table_name = 'graph_overlays'"); + + var attestationsTableExists = await connection.ExecuteScalarAsync( + @"SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema = 'vex' + AND table_name = 'attestations'"); + var demoLinksets = await connection.ExecuteScalarAsync( "SELECT COUNT(*) FROM vex.linksets WHERE tenant = 'demo-prod'"); @@ -260,6 +262,8 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime "SELECT COUNT(*) FROM excititor.source_trust_vectors WHERE tenant = 'demo-prod'"); claimsTableExists.Should().Be(1, "the persisted claim store must own a real PostgreSQL table"); + graphOverlaysTableExists.Should().Be(1, "the graph overlay store must own a real PostgreSQL table"); + attestationsTableExists.Should().Be(1, "the attestation store must own a real PostgreSQL table"); demoLinksets.Should().Be(0, "startup migrations must not seed demo linksets into the live schema"); demoRawDocuments.Should().Be(0, "startup migrations must not seed demo raw VEX documents"); demoSourceVectors.Should().Be(0, "startup migrations must not seed demo trust vectors"); @@ -267,64 +271,16 @@ public sealed class ExcititorMigrationTests : IAsyncLifetime private async Task ApplyAllMigrationsAsync(string connectionString) { - await using var connection = new NpgsqlConnection(connectionString); - await connection.OpenAsync(); + var runner = new MigrationRunner( + connectionString, + ExcititorDataSource.DefaultSchemaName, + "Excititor", + NullLogger.Instance); - // Create migration tracking table - await connection.ExecuteAsync(@" - CREATE TABLE IF NOT EXISTS __migrations ( - id SERIAL PRIMARY KEY, - migration_id TEXT NOT NULL UNIQUE, - applied_at TIMESTAMPTZ DEFAULT NOW() - )"); - - // Get and apply all migrations - var migrationFiles = GetMigrationFiles(); - - foreach (var migrationFile in migrationFiles.OrderBy(f => f)) - { - var migrationId = Path.GetFileName(migrationFile); - - // Skip if already applied - var alreadyApplied = await connection.ExecuteScalarAsync( - "SELECT COUNT(*) FROM __migrations WHERE migration_id = @Id", - new { Id = migrationId }); - - if (alreadyApplied > 0) - continue; - - // Apply migration - var sql = GetMigrationContent(migrationFile); - if (!string.IsNullOrWhiteSpace(sql)) - { - await connection.ExecuteAsync(sql); - await connection.ExecuteAsync( - "INSERT INTO __migrations (migration_id) VALUES (@Id)", - new { Id = migrationId }); - } - } - } - - private static IEnumerable GetMigrationFiles() - { - var assembly = typeof(ExcititorDataSource).Assembly; - var resourceNames = assembly.GetManifestResourceNames() - .Where(n => n.Contains("Migrations") && n.EndsWith(".sql")) - .OrderBy(n => n); - - return resourceNames; - } - - private static string GetMigrationContent(string resourceName) - { - var assembly = typeof(ExcititorDataSource).Assembly; - using var stream = assembly.GetManifestResourceStream(resourceName); - if (stream == null) - return string.Empty; - - using var reader = new StreamReader(stream); - return reader.ReadToEnd(); + await runner.RunFromAssemblyAsync( + typeof(ExcititorDataSource).Assembly, + resourcePrefix: null, + options: null, + cancellationToken: default); } } - - diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresGraphOverlayStoreTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresGraphOverlayStoreTests.cs new file mode 100644 index 000000000..9b80d8d17 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresGraphOverlayStoreTests.cs @@ -0,0 +1,119 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Persistence.Postgres; +using StellaOps.Excititor.WebService.Contracts; +using StellaOps.Excititor.WebService.Services; +using StellaOps.Infrastructure.Postgres.Options; +using StellaOps.TestKit; +using Xunit; + +namespace StellaOps.Excititor.Persistence.Tests; + +[Collection(ExcititorPostgresCollection.Name)] +public sealed class PostgresGraphOverlayStoreTests : IAsyncLifetime +{ + private readonly ExcititorPostgresFixture _fixture; + private readonly string _tenantId = "tenant-" + Guid.NewGuid().ToString("N")[..8]; + private ExcititorDataSource _dataSource = null!; + private PostgresGraphOverlayStore _store = null!; + + public PostgresGraphOverlayStoreTests(ExcititorPostgresFixture fixture) + { + _fixture = fixture; + } + + public async ValueTask InitializeAsync() + { + await _fixture.Fixture.RunMigrationsFromAssemblyAsync( + typeof(ExcititorDataSource).Assembly, + moduleName: "Excititor", + resourcePrefix: "Migrations", + cancellationToken: CancellationToken.None); + + await _fixture.TruncateAllTablesAsync(); + InitializeStore(); + } + + public async ValueTask DisposeAsync() + { + if (_dataSource is not null) + { + await _dataSource.DisposeAsync(); + } + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task SaveAsync_AndFindByPurlsAsync_SurviveStoreRecreation() + { + var initial = CreateOverlay("ADV-1", "not_affected", DateTimeOffset.UtcNow.AddMinutes(-5)); + var updated = CreateOverlay("ADV-1", "under_investigation", DateTimeOffset.UtcNow); + + await _store.SaveAsync(_tenantId, new[] { initial }, CancellationToken.None); + await _store.SaveAsync(_tenantId, new[] { updated }, CancellationToken.None); + await _dataSource.DisposeAsync(); + + InitializeStore(); + + var results = await _store.FindByPurlsAsync(_tenantId, new[] { "pkg:npm/example@1.0.0" }, CancellationToken.None); + + results.Should().ContainSingle(); + results[0].Status.Should().Be("under_investigation"); + results[0].AdvisoryId.Should().Be("ADV-1"); + } + + [Trait("Category", TestCategories.Unit)] + [Fact] + public async Task FindWithConflictsAsync_ReturnsOnlyConflictedRows() + { + var conflicted = CreateOverlay("ADV-conflict", "affected", DateTimeOffset.UtcNow, hasConflict: true); + var clean = CreateOverlay("ADV-clean", "not_affected", DateTimeOffset.UtcNow.AddMinutes(-1)); + + await _store.SaveAsync(_tenantId, new[] { conflicted, clean }, CancellationToken.None); + + var results = await _store.FindWithConflictsAsync(_tenantId, limit: 10, CancellationToken.None); + + results.Should().ContainSingle(); + results[0].AdvisoryId.Should().Be("ADV-conflict"); + } + + private void InitializeStore() + { + var options = Options.Create(new PostgresOptions + { + ConnectionString = _fixture.ConnectionString, + SchemaName = _fixture.SchemaName, + AutoMigrate = false + }); + + _dataSource = new ExcititorDataSource(options, NullLogger.Instance); + _store = new PostgresGraphOverlayStore(_dataSource, NullLogger.Instance); + } + + private GraphOverlayItem CreateOverlay(string advisoryId, string status, DateTimeOffset generatedAt, bool hasConflict = false) + => new( + "1.0.0", + generatedAt, + _tenantId, + "pkg:npm/example@1.0.0", + advisoryId, + "provider-a", + status, + Array.Empty(), + hasConflict + ? new[] { new GraphOverlayConflict("status", "provider_disagreement", new[] { "affected", "not_affected" }, new[] { "provider-a", "provider-b" }) } + : Array.Empty(), + new[] + { + new GraphOverlayObservation("obs-1", "sha256:hash-1", generatedAt) + }, + new GraphOverlayProvenance( + "linkset-1", + "sha256:linkset-1", + new[] { "sha256:hash-1" }, + null, + null, + hasConflict ? "conflict-plan" : null), + null); +} diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresVexAttestationStoreTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresVexAttestationStoreTests.cs index d8d6e9c32..5711b0375 100644 --- a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresVexAttestationStoreTests.cs +++ b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/PostgresVexAttestationStoreTests.cs @@ -35,12 +35,6 @@ public sealed class PostgresVexAttestationStoreTests : IAsyncLifetime public async ValueTask InitializeAsync() { - await _fixture.Fixture.RunMigrationsFromAssemblyAsync( - typeof(ExcititorDataSource).Assembly, - moduleName: "Excititor", - resourcePrefix: "Migrations", - cancellationToken: CancellationToken.None); - await _fixture.TruncateAllTablesAsync(); } @@ -205,4 +199,3 @@ public sealed class PostgresVexAttestationStoreTests : IAsyncLifetime } - diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/StellaOps.Excititor.Persistence.Tests.csproj b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/StellaOps.Excititor.Persistence.Tests.csproj index e6394182d..7302166b0 100644 --- a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/StellaOps.Excititor.Persistence.Tests.csproj +++ b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/StellaOps.Excititor.Persistence.Tests.csproj @@ -21,7 +21,8 @@ + - \ No newline at end of file + diff --git a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/TASKS.md index dd90728ae..c446b9450 100644 --- a/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Excititor.Persistence.Tests/TASKS.md @@ -9,3 +9,5 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0324-T | DONE | Revalidated 2026-01-07; test coverage audit for Excititor.Persistence.Tests. | | AUDIT-0324-A | DONE | Waived (test project; revalidated 2026-01-07). | | QA-DEVOPS-VERIFY-002-T | DONE | 2026-02-11: Added Rekor linkage behavioral tests (round-trip, pending ordering, missing-observation negative path) for `vex-rekor-linkage` run-001. | +| REALPLAN-007-A | DONE | 2026-04-15: Added the durable attestation migration and reran `PostgresVexAttestationStoreTests` 7/7 plus `ExcititorMigrationTests` 8/8 to prove restart-survival reads. | +| REALPLAN-007-B | DONE | 2026-04-15: Added `PostgresGraphOverlayStoreTests` plus migration coverage for durable `vex.graph_overlays` persistence and restart-survival query behavior. | diff --git a/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/AttestationStoreWiringTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/AttestationStoreWiringTests.cs new file mode 100644 index 000000000..d5ec47e65 --- /dev/null +++ b/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/AttestationStoreWiringTests.cs @@ -0,0 +1,25 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Core.Evidence; +using StellaOps.Excititor.Core.Storage; +using StellaOps.Excititor.Persistence.Postgres.Repositories; +using Xunit; + +using StellaOps.TestKit; +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class AttestationStoreWiringTests +{ + [Trait("Category", TestCategories.Unit)] + [Fact] + public void Host_Resolves_PostgresAttestationStore() + { + using var factory = new TestWebApplicationFactory(); + using var scope = factory.Services.CreateScope(); + + var store = scope.ServiceProvider.GetRequiredService(); + + store.Should().BeOfType(); + store.Should().NotBeOfType(); + } +} diff --git a/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/GraphOverlayStoreTests.cs b/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/GraphOverlayStoreTests.cs index b3e260b63..e25b2bd2b 100644 --- a/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/GraphOverlayStoreTests.cs +++ b/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/GraphOverlayStoreTests.cs @@ -1,4 +1,5 @@ -using StellaOps.Excititor.WebService.Contracts; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; using StellaOps.Excititor.WebService.Services; using Xunit; @@ -8,46 +9,15 @@ namespace StellaOps.Excititor.WebService.Tests; public sealed class GraphOverlayStoreTests { [Trait("Category", TestCategories.Unit)] - [Fact] - public async Task SaveAndFindByPurls_ReturnsLatestPerSourceAdvisory() + [Fact] + public void Host_Resolves_PostgresGraphOverlayStore() { - var store = new InMemoryGraphOverlayStore(); - var overlays = new[] - { - new GraphOverlayItem( - SchemaVersion: "1.0.0", - GeneratedAt: DateTimeOffset.UtcNow.AddMinutes(-1), - Tenant: "tenant-a", - Purl: "pkg:npm/example@1.0.0", - AdvisoryId: "ADV-1", - Source: "provider-a", - Status: "not_affected", - Summary: new GraphOverlaySummary(0, 1, 0, 0), - Justifications: Array.Empty(), - Conflicts: Array.Empty(), - Observations: Array.Empty(), - Provenance: new GraphOverlayProvenance("tenant-a", new[] { "provider-a" }, new[] { "ADV-1" }, new[] { "pkg:npm/example@1.0.0" }, Array.Empty(), Array.Empty()), - Cache: null), - new GraphOverlayItem( - SchemaVersion: "1.0.0", - GeneratedAt: DateTimeOffset.UtcNow, - Tenant: "tenant-a", - Purl: "pkg:npm/example@1.0.0", - AdvisoryId: "ADV-1", - Source: "provider-a", - Status: "under_investigation", - Summary: new GraphOverlaySummary(0, 0, 1, 0), - Justifications: Array.Empty(), - Conflicts: Array.Empty(), - Observations: Array.Empty(), - Provenance: new GraphOverlayProvenance("tenant-a", new[] { "provider-a" }, new[] { "ADV-1" }, new[] { "pkg:npm/example@1.0.0" }, Array.Empty(), Array.Empty()), - Cache: null) - }; + using var factory = new TestWebApplicationFactory(); + using var scope = factory.Services.CreateScope(); - await store.SaveAsync("tenant-a", overlays, CancellationToken.None); - var results = await store.FindByPurlsAsync("tenant-a", new[] { "pkg:npm/example@1.0.0" }, CancellationToken.None); + var store = scope.ServiceProvider.GetRequiredService(); - var single = Assert.Single(results); - Assert.Equal("under_investigation", single.Status); + store.Should().BeOfType(); + store.Should().NotBeOfType(); } } diff --git a/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj b/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj index bae0436a4..ac1f4b9e3 100644 --- a/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj +++ b/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj @@ -30,11 +30,13 @@ + + diff --git a/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/TASKS.md b/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/TASKS.md index b3033b798..4fa3eb8cd 100644 --- a/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/TASKS.md +++ b/src/Concelier/__Tests/StellaOps.Excititor.WebService.Tests/TASKS.md @@ -8,3 +8,4 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229 | AUDIT-0328-M | DONE | Revalidated 2026-01-07; maintainability audit for Excititor.WebService.Tests. | | AUDIT-0328-T | DONE | Revalidated 2026-01-07; test coverage audit for Excititor.WebService.Tests. | | AUDIT-0328-A | DONE | Waived (test project; revalidated 2026-01-07). | +| REALPLAN-007-B | DONE | 2026-04-15: Added host wiring proof that Excititor.WebService resolves `IGraphOverlayStore` to `PostgresGraphOverlayStore` instead of the in-memory fallback. |