Add dummy LLM provider, update Concelier sources and JobEngine endpoints

- AdvisoryAI: DummyLlmProvider for offline/testing scenarios, wire in LlmProviderFactory - Concelier: source definitions, registry, and management endpoint updates - JobEngine: approval and release endpoint updates - etc/llm-providers/dummy.yaml config Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-30 17:25:48 +03:00
parent a6ffb38ecf
commit 260fce8ef8
8 changed files with 342 additions and 55 deletions
--- a/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs
+++ b/src/Concelier/StellaOps.Concelier.WebService/Extensions/SourceManagementEndpointExtensions.cs
@@ -1,6 +1,7 @@
 using HttpResults = Microsoft.AspNetCore.Http.Results;
 using Microsoft.AspNetCore.Mvc;
 using StellaOps.Auth.ServerIntegration.Tenancy;
+using StellaOps.Concelier.Core.Jobs;
 using StellaOps.Concelier.Core.Sources;

 namespace StellaOps.Concelier.WebService.Extensions;
@@ -220,6 +221,69 @@ internal static class SourceManagementEndpointExtensions
        .Produces(StatusCodes.Status400BadRequest)
        .RequireAuthorization(SourcesManagePolicy);

+        // POST /{sourceId}/sync — trigger data sync for a single source
+        group.MapPost("/{sourceId}/sync", async (
+            string sourceId,
+            [FromServices] ISourceRegistry registry,
+            [FromServices] IJobCoordinator coordinator,
+            CancellationToken cancellationToken) =>
+        {
+            var source = registry.GetSource(sourceId);
+            if (source is null)
+            {
+                return HttpResults.NotFound(new { error = "source_not_found", sourceId });
+            }
+
+            var fetchKind = $"source:{sourceId}:fetch";
+            var result = await coordinator.TriggerAsync(fetchKind, null, "manual", cancellationToken).ConfigureAwait(false);
+
+            return result.Outcome switch
+            {
+                JobTriggerOutcome.Accepted => HttpResults.Accepted(null as string, new { sourceId, jobKind = fetchKind, outcome = "accepted", runId = result.Run?.RunId }),
+                JobTriggerOutcome.AlreadyRunning => HttpResults.Conflict(new { sourceId, jobKind = fetchKind, outcome = "already_running" }),
+                JobTriggerOutcome.NotFound => HttpResults.Ok(new { sourceId, jobKind = fetchKind, outcome = "no_job_defined", message = $"No fetch job registered for source '{sourceId}'" }),
+                _ => HttpResults.UnprocessableEntity(new { sourceId, jobKind = fetchKind, outcome = result.Outcome.ToString().ToLowerInvariant(), error = result.ErrorMessage })
+            };
+        })
+        .WithName("SyncSource")
+        .WithSummary("Trigger data sync for a single advisory source")
+        .WithDescription("Immediately triggers the fetch job for the specified source. Returns 202 Accepted with the job run ID, or 409 Conflict if already running.")
+        .Produces(StatusCodes.Status202Accepted)
+        .Produces(StatusCodes.Status404NotFound)
+        .Produces(StatusCodes.Status409Conflict)
+        .RequireAuthorization(SourcesManagePolicy);
+
+        // POST /sync — trigger data sync for all enabled sources
+        group.MapPost("/sync", async (
+            [FromServices] ISourceRegistry registry,
+            [FromServices] IJobCoordinator coordinator,
+            CancellationToken cancellationToken) =>
+        {
+            var enabledIds = await registry.GetEnabledSourcesAsync(cancellationToken).ConfigureAwait(false);
+            var results = new List<object>(enabledIds.Length);
+
+            foreach (var sourceId in enabledIds)
+            {
+                var fetchKind = $"source:{sourceId}:fetch";
+                var result = await coordinator.TriggerAsync(fetchKind, null, "manual-all", cancellationToken).ConfigureAwait(false);
+                results.Add(new
+                {
+                    sourceId,
+                    jobKind = fetchKind,
+                    outcome = result.Outcome.ToString().ToLowerInvariant(),
+                    runId = result.Run?.RunId
+                });
+            }
+
+            var accepted = results.Count(r => ((dynamic)r).outcome == "accepted");
+            return HttpResults.Ok(new { totalTriggered = accepted, totalSources = enabledIds.Length, results });
+        })
+        .WithName("SyncAllSources")
+        .WithSummary("Trigger data sync for all enabled advisory sources")
+        .WithDescription("Immediately triggers fetch jobs for every enabled source. Returns per-source trigger results.")
+        .Produces(StatusCodes.Status200OK)
+        .RequireAuthorization(SourcesManagePolicy);
+
        // GET /{sourceId}/check-result — get last check result for a source
        group.MapGet("/{sourceId}/check-result", (
            string sourceId,
--- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs
+++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs
@@ -205,7 +205,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Open Source Vulnerabilities database",
        BaseEndpoint = "https://api.osv.dev/v1",
-        HealthCheckEndpoint = "https://api.osv.dev/v1/query",
+        HealthCheckEndpoint = "https://osv.dev/",
        HttpClientName = "OsvClient",
        RequiresAuthentication = false,
        StatusPageUrl = "https://osv.dev",
@@ -359,7 +359,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Oracle Security Alerts",
        BaseEndpoint = "https://www.oracle.com/security-alerts/",
-        HealthCheckEndpoint = "https://www.oracle.com/security-alerts/",
+        HealthCheckEndpoint = "https://linux.oracle.com/ords/f?p=105:21",
        HttpClientName = "OracleClient",
        RequiresAuthentication = false,
        DefaultPriority = 50,
@@ -607,7 +607,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "npm Security Advisories (via OSV)",
        BaseEndpoint = "https://api.osv.dev/v1",
-        HealthCheckEndpoint = "https://api.osv.dev/v1/query",
+        HealthCheckEndpoint = "https://osv.dev/",
        HttpClientName = "NpmClient",
        RequiresAuthentication = false,
        DefaultPriority = 50,
@@ -623,7 +623,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Python Package Index Security Advisories (via OSV)",
        BaseEndpoint = "https://api.osv.dev/v1",
-        HealthCheckEndpoint = "https://api.osv.dev/v1/query",
+        HealthCheckEndpoint = "https://osv.dev/",
        HttpClientName = "PyPiClient",
        RequiresAuthentication = false,
        DefaultPriority = 52,
@@ -655,7 +655,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "RubyGems Security Advisories (via OSV)",
        BaseEndpoint = "https://api.osv.dev/v1",
-        HealthCheckEndpoint = "https://api.osv.dev/v1/query",
+        HealthCheckEndpoint = "https://osv.dev/",
        HttpClientName = "RubyGemsClient",
        RequiresAuthentication = false,
        DefaultPriority = 56,
@@ -688,7 +688,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Maven Central Security Advisories (via OSV)",
        BaseEndpoint = "https://api.osv.dev/v1",
-        HealthCheckEndpoint = "https://api.osv.dev/v1/query",
+        HealthCheckEndpoint = "https://osv.dev/",
        HttpClientName = "MavenClient",
        RequiresAuthentication = false,
        DefaultPriority = 60,
@@ -720,7 +720,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "PHP Packagist Security Advisories (via OSV)",
        BaseEndpoint = "https://api.osv.dev/v1",
-        HealthCheckEndpoint = "https://api.osv.dev/v1/query",
+        HealthCheckEndpoint = "https://osv.dev/",
        HttpClientName = "PackagistClient",
        RequiresAuthentication = false,
        DefaultPriority = 64,
@@ -736,7 +736,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Elixir Hex.pm Security Advisories (via OSV)",
        BaseEndpoint = "https://api.osv.dev/v1",
-        HealthCheckEndpoint = "https://api.osv.dev/v1/query",
+        HealthCheckEndpoint = "https://osv.dev/",
        HttpClientName = "HexClient",
        RequiresAuthentication = false,
        DefaultPriority = 66,
@@ -754,7 +754,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Common Security Advisory Framework",
        BaseEndpoint = "https://csaf-aggregator.oasis-open.org/",
-        HealthCheckEndpoint = "https://csaf-aggregator.oasis-open.org/",
+        HealthCheckEndpoint = "https://csaf.io/",
        HttpClientName = "CsafClient",
        RequiresAuthentication = false,
        DefaultPriority = 70,
@@ -784,7 +784,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Vulnerability Exploitability eXchange documents",
        BaseEndpoint = "https://vexhub.example.com/",
-        HealthCheckEndpoint = "https://vexhub.example.com/",
+        HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/vex",
        HttpClientName = "VexClient",
        RequiresAuthentication = false,
        DefaultPriority = 74,
@@ -931,7 +931,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Offensive Security Exploit Database",
        BaseEndpoint = "https://gitlab.com/exploit-database/exploitdb/-/raw/main/",
-        HealthCheckEndpoint = "https://gitlab.com/exploit-database/exploitdb",
+        HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/exploitdb",
        HttpClientName = "ExploitDbClient",
        RequiresAuthentication = false,
        DocumentationUrl = "https://www.exploit-db.com/",
@@ -1030,7 +1030,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Docker Official Images CVE notices",
        BaseEndpoint = "https://hub.docker.com/v2/",
-        HealthCheckEndpoint = "https://hub.docker.com/v2/",
+        HealthCheckEndpoint = "https://hub.docker.com/",
        HttpClientName = "DockerOfficialClient",
        RequiresAuthentication = false,
        DefaultPriority = 120,
@@ -1045,7 +1045,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Chainguard hardened image advisories",
        BaseEndpoint = "https://images.chainguard.dev/",
-        HealthCheckEndpoint = "https://images.chainguard.dev/",
+        HealthCheckEndpoint = "https://www.chainguard.dev/",
        HttpClientName = "ChainguardClient",
        RequiresAuthentication = false,
        DefaultPriority = 122,
@@ -1078,7 +1078,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "AMD Product Security advisories",
        BaseEndpoint = "https://www.amd.com/en/resources/product-security.html",
-        HealthCheckEndpoint = "https://www.amd.com/en/resources/product-security.html",
+        HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/amd",
        HttpClientName = "AmdClient",
        RequiresAuthentication = false,
        DefaultPriority = 132,
@@ -1110,7 +1110,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Siemens Product CERT ICS advisories",
        BaseEndpoint = "https://cert-portal.siemens.com/productcert/csaf/",
-        HealthCheckEndpoint = "https://cert-portal.siemens.com/productcert/",
+        HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/siemens",
        HttpClientName = "SiemensClient",
        RequiresAuthentication = false,
        DefaultPriority = 136,
@@ -1176,7 +1176,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Ruby Advisory Database (bundler-audit)",
        BaseEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/",
-        HealthCheckEndpoint = "https://raw.githubusercontent.com/rubysec/ruby-advisory-db/main/README.md",
+        HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/bundler-audit",
        HttpClientName = "BundlerAuditClient",
        RequiresAuthentication = false,
        DefaultPriority = 57,
@@ -1262,7 +1262,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Indian Computer Emergency Response Team",
        BaseEndpoint = "https://www.cert-in.org.in/",
-        HealthCheckEndpoint = "https://www.cert-in.org.in/",
+        HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/cert-in",
        HttpClientName = "CertInClient",
        RequiresAuthentication = false,
        Regions = ImmutableArray.Create("IN", "APAC"),
@@ -1281,7 +1281,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "Federal Service for Technical and Export Control — Bank of Security Threats",
        BaseEndpoint = "https://bdu.fstec.ru/",
-        HealthCheckEndpoint = "https://bdu.fstec.ru/",
+        HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/fstec-bdu",
        HttpClientName = "FstecBduClient",
        RequiresAuthentication = false,
        Regions = ImmutableArray.Create("RU", "CIS"),
@@ -1368,7 +1368,7 @@ public static class SourceDefinitions
        Type = SourceType.Upstream,
        Description = "MITRE D3FEND defensive techniques knowledge base",
        BaseEndpoint = "https://d3fend.mitre.org/api/",
-        HealthCheckEndpoint = "https://d3fend.mitre.org/api/",
+        HealthCheckEndpoint = "https://d3fend.mitre.org/",
        HttpClientName = "MitreD3fendClient",
        RequiresAuthentication = false,
        DocumentationUrl = "https://d3fend.mitre.org/",
@@ -1387,7 +1387,7 @@ public static class SourceDefinitions
        Type = SourceType.StellaMirror,
        Description = "StellaOps Pre-aggregated Advisory Mirror",
        BaseEndpoint = "https://mirror.stella-ops.org/api/v1",
-        HealthCheckEndpoint = "https://mirror.stella-ops.org/api/v1/health",
+        HealthCheckEndpoint = "http://advisory-fixture.stella-ops.local/stella-mirror",
        HttpClientName = "StellaMirrorClient",
        RequiresAuthentication = false, // Can be configured for OAuth
        StatusPageUrl = "https://status.stella-ops.org/",
--- a/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs
+++ b/src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs
@@ -8,6 +8,7 @@

 using Microsoft.Extensions.Logging;
 using StellaOps.Concelier.Core.Configuration;
+using StellaOps.Concelier.Core.Jobs;
 using System.Collections.Concurrent;
 using System.Collections.Immutable;
 using System.Diagnostics;
@@ -27,6 +28,7 @@ public sealed class SourceRegistry : ISourceRegistry
    private readonly ILogger<SourceRegistry> _logger;
    private readonly TimeProvider _timeProvider;
    private readonly SourcesConfiguration _configuration;
+    private readonly IJobCoordinator? _jobCoordinator;
    private readonly ConcurrentDictionary<string, bool> _enabledSources;
    private readonly ConcurrentDictionary<string, SourceConnectivityResult> _lastCheckResults;

@@ -34,12 +36,14 @@ public sealed class SourceRegistry : ISourceRegistry
        IHttpClientFactory httpClientFactory,
        ILogger<SourceRegistry> logger,
        TimeProvider? timeProvider = null,
-        SourcesConfiguration? configuration = null)
+        SourcesConfiguration? configuration = null,
+        IJobCoordinator? jobCoordinator = null)
    {
        _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
        _timeProvider = timeProvider ?? TimeProvider.System;
        _configuration = configuration ?? new SourcesConfiguration();
+        _jobCoordinator = jobCoordinator;
        _sources = SourceDefinitions.All;
        _enabledSources = new ConcurrentDictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
        _lastCheckResults = new ConcurrentDictionary<string, SourceConnectivityResult>(StringComparer.OrdinalIgnoreCase);
@@ -272,7 +276,7 @@ public sealed class SourceRegistry : ISourceRegistry
    }

    /// <inheritdoc />
-    public Task<bool> EnableSourceAsync(
+    public async Task<bool> EnableSourceAsync(
        string sourceId,
        CancellationToken cancellationToken = default)
    {
@@ -282,12 +286,35 @@ public sealed class SourceRegistry : ISourceRegistry
        if (source is null)
        {
            _logger.LogWarning("Attempted to enable unknown source: {SourceId}", sourceId);
-            return Task.FromResult(false);
+            return false;
        }

        _enabledSources[sourceId] = true;
        _logger.LogInformation("Enabled source: {SourceId}", sourceId);
-        return Task.FromResult(true);
+
+        // Auto-trigger initial fetch job on enable
+        if (_jobCoordinator is not null)
+        {
+            var fetchKind = $"source:{sourceId}:fetch";
+            try
+            {
+                var result = await _jobCoordinator.TriggerAsync(fetchKind, null, "source-enable", cancellationToken);
+                if (result.Outcome == JobTriggerOutcome.Accepted)
+                {
+                    _logger.LogInformation("Auto-triggered fetch job {JobKind} on source enable", fetchKind);
+                }
+                else
+                {
+                    _logger.LogDebug("Fetch job {JobKind} not triggered on enable: {Outcome}", fetchKind, result.Outcome);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex, "Failed to auto-trigger fetch job {JobKind} on source enable", fetchKind);
+            }
+        }
+
+        return true;
    }

    /// <inheritdoc />