diff --git a/devops/compose/README.md b/devops/compose/README.md index 8e97b33e6..0b30010a5 100644 --- a/devops/compose/README.md +++ b/devops/compose/README.md @@ -313,13 +313,15 @@ export VAULT_TOKEN=stellaops-dev-root-token-2026 vault kv put secret/harbor robot-account="harbor-robot-token" vault kv put secret/github app-private-key="your-key" vault kv put secret/gitea api-token="your-gitea-token" -vault kv put secret/gitlab access-token="glpat-your-token" +vault kv put secret/gitlab access-token="glpat-your-token" registry-basic="root:glpat-your-token" vault kv put secret/jenkins api-token="user:token" vault kv put secret/nexus admin-password="your-password" ``` Gitea is now bootstrapped by the compose service itself: a fresh `stellaops-gitea-data` volume creates the default local admin user and the repository root before the container reports healthy. Personal access tokens remain a manual step because Gitea only reveals the token value when it is created. +When you enable the optional GitLab registry surface (`GITLAB_ENABLE_REGISTRY=true`), register it through the `GitLabContainerRegistry` provider with `authref://vault/gitlab#registry-basic`. The local Docker registry connector now follows the registry's Bearer challenge and exchanges that `username:personal-access-token` secret against `jwt/auth` before retrying catalog and tag probes. + `docker-compose.testing.yml` is a separate infrastructure-test lane. It starts `postgres-test`, `valkey-test`, mocks, and an isolated Gitea profile on different ports; it does not start Consul or GitLab. Use `docker-compose.integrations.yml` only when you need real third-party providers for connector validation. **Backend connector plugins** (8 total, loaded in Integrations service): diff --git a/devops/compose/postgres-init/01-create-schemas.sql b/devops/compose/postgres-init/01-create-schemas.sql index 6758b89e7..acfe355da 100644 --- a/devops/compose/postgres-init/01-create-schemas.sql +++ b/devops/compose/postgres-init/01-create-schemas.sql @@ -10,6 +10,7 @@ CREATE SCHEMA IF NOT EXISTS notify; CREATE SCHEMA IF NOT EXISTS notifier; CREATE SCHEMA IF NOT EXISTS evidence; CREATE SCHEMA IF NOT EXISTS findings; +CREATE SCHEMA IF NOT EXISTS graph; CREATE SCHEMA IF NOT EXISTS timeline; CREATE SCHEMA IF NOT EXISTS doctor; CREATE SCHEMA IF NOT EXISTS issuer_directory; diff --git a/devops/compose/postgres-init/02-findings-ledger-tables.sql b/devops/compose/postgres-init/02-findings-ledger-tables.sql index b92a754ac..c378e831c 100644 --- a/devops/compose/postgres-init/02-findings-ledger-tables.sql +++ b/devops/compose/postgres-init/02-findings-ledger-tables.sql @@ -563,3 +563,86 @@ BEGIN END $$; COMMENT ON TABLE ledger_snapshots IS 'Point-in-time snapshots of ledger state for time-travel queries'; + +-- ============================================================================ +-- 010_vex_fix_audit_tables.sql - VulnExplorer persistence tables +-- ============================================================================ + +BEGIN; + +CREATE TABLE IF NOT EXISTS vex_decisions ( + id UUID NOT NULL, + tenant_id TEXT NOT NULL, + vulnerability_id TEXT NOT NULL, + subject_type TEXT NOT NULL, + subject_name TEXT NOT NULL, + subject_digest JSONB NOT NULL DEFAULT '{}'::JSONB, + subject_sbom_node_id TEXT, + status TEXT NOT NULL, + justification_type TEXT NOT NULL, + justification_text TEXT, + evidence_refs JSONB, + scope_environments TEXT[], + scope_projects TEXT[], + valid_not_before TIMESTAMPTZ, + valid_not_after TIMESTAMPTZ, + attestation_ref_id TEXT, + attestation_ref_digest JSONB, + attestation_ref_storage TEXT, + signed_override JSONB, + supersedes_decision_id UUID, + created_by_id TEXT NOT NULL, + created_by_name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ, + CONSTRAINT pk_vex_decisions PRIMARY KEY (tenant_id, id) +) PARTITION BY LIST (tenant_id); + +CREATE TABLE IF NOT EXISTS vex_decisions_default PARTITION OF vex_decisions DEFAULT; + +CREATE INDEX IF NOT EXISTS ix_vex_decisions_vuln + ON vex_decisions (tenant_id, vulnerability_id, created_at DESC); +CREATE INDEX IF NOT EXISTS ix_vex_decisions_status + ON vex_decisions (tenant_id, status); +CREATE INDEX IF NOT EXISTS ix_vex_decisions_subject + ON vex_decisions (tenant_id, subject_name); + +CREATE TABLE IF NOT EXISTS fix_verifications ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + tenant_id TEXT NOT NULL, + cve_id TEXT NOT NULL, + component_purl TEXT NOT NULL, + artifact_digest TEXT, + verdict TEXT NOT NULL DEFAULT 'pending', + transitions JSONB NOT NULL DEFAULT '[]'::JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT pk_fix_verifications PRIMARY KEY (tenant_id, id), + CONSTRAINT uq_fix_verifications_cve UNIQUE (tenant_id, cve_id) +) PARTITION BY LIST (tenant_id); + +CREATE TABLE IF NOT EXISTS fix_verifications_default PARTITION OF fix_verifications DEFAULT; + +CREATE INDEX IF NOT EXISTS ix_fix_verifications_cve + ON fix_verifications (tenant_id, cve_id); +CREATE INDEX IF NOT EXISTS ix_fix_verifications_verdict + ON fix_verifications (tenant_id, verdict); + +CREATE TABLE IF NOT EXISTS audit_bundles ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + tenant_id TEXT NOT NULL, + bundle_id TEXT NOT NULL, + decision_ids UUID[] NOT NULL, + attestation_digest TEXT, + evidence_refs TEXT[] NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT pk_audit_bundles PRIMARY KEY (tenant_id, id), + CONSTRAINT uq_audit_bundles_bundle_id UNIQUE (tenant_id, bundle_id) +) PARTITION BY LIST (tenant_id); + +CREATE TABLE IF NOT EXISTS audit_bundles_default PARTITION OF audit_bundles DEFAULT; + +CREATE INDEX IF NOT EXISTS ix_audit_bundles_created + ON audit_bundles (tenant_id, created_at DESC); + +COMMIT; diff --git a/devops/compose/postgres-init/03-scheduler-tables.sql b/devops/compose/postgres-init/03-scheduler-tables.sql index 63c88ded6..3ce7885f5 100644 --- a/devops/compose/postgres-init/03-scheduler-tables.sql +++ b/devops/compose/postgres-init/03-scheduler-tables.sql @@ -690,3 +690,26 @@ ALTER TABLE scheduler.scheduler_exceptions FORCE ROW LEVEL SECURITY; CREATE POLICY scheduler_exceptions_tenant_isolation ON scheduler.scheduler_exceptions FOR ALL USING (tenant_id = scheduler_app.require_current_tenant()) WITH CHECK (tenant_id = scheduler_app.require_current_tenant()); + +-- ============================================================================ +-- failure_signatures - Scheduler failure signature tracking +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS scheduler.failure_signatures ( + signature_id UUID NOT NULL, + tenant_id TEXT NOT NULL, + signature_key TEXT NOT NULL, + pattern TEXT NOT NULL, + severity TEXT NOT NULL DEFAULT 'medium', + description TEXT, + remediation TEXT, + auto_retry BOOLEAN NOT NULL DEFAULT false, + max_retries INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + created_by TEXT, + CONSTRAINT failure_signatures_pkey PRIMARY KEY (signature_id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_scheduler_failure_signatures_key + ON scheduler.failure_signatures(tenant_id, signature_key); diff --git a/devops/compose/router-gateway-local.json b/devops/compose/router-gateway-local.json index 4f584128f..2f088d3aa 100644 --- a/devops/compose/router-gateway-local.json +++ b/devops/compose/router-gateway-local.json @@ -84,6 +84,16 @@ { "Type": "Microservice", "Path": "^/api/v1/release-control(.*)", "IsRegex": true, "TranslatesTo": "http://platform.stella-ops.local/api/v1/release-control$1" }, { "Type": "Microservice", "Path": "^/api/v1/gateway/rate-limits(.*)", "IsRegex": true, "TranslatesTo": "http://platform.stella-ops.local/api/v1/gateway/rate-limits$1" }, { "Type": "Microservice", "Path": "^/api/v1/jobengine/quotas(.*)", "IsRegex": true, "TranslatesTo": "http://platform.stella-ops.local/api/v1/jobengine/quotas$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/registry/packs(.*)", "IsRegex": true, "TranslatesTo": "http://packsregistry.stella-ops.local/api/v1/packs$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/deadletter(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/deadletter$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/jobs(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/jobs$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/runs(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/runs$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/dag(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/dag$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/pack-runs(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/pack-runs$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/stream(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/stream$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/audit(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/audit$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/sources(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/sources$1" }, + { "Type": "Microservice", "Path": "^/api/v1/jobengine/slos(.*)", "IsRegex": true, "TranslatesTo": "http://release-orchestrator.stella-ops.local/api/v1/jobengine/slos$1" }, { "Type": "Microservice", "Path": "^/api/v1/reachability(.*)", "IsRegex": true, "TranslatesTo": "http://reachgraph.stella-ops.local/api/v1/reachability$1" }, { "Type": "Microservice", "Path": "^/api/v1/timeline(.*)", "IsRegex": true, "TranslatesTo": "http://timeline.stella-ops.local/api/v1/timeline$1" }, { "Type": "Microservice", "Path": "^/api/v1/audit(.*)", "IsRegex": true, "TranslatesTo": "http://timeline.stella-ops.local/api/v1/audit$1" }, diff --git a/src/Attestor/__Libraries/StellaOps.Attestor.Persistence/StellaOps.Attestor.Persistence.csproj b/src/Attestor/__Libraries/StellaOps.Attestor.Persistence/StellaOps.Attestor.Persistence.csproj index 24c410d39..f2701bf09 100644 --- a/src/Attestor/__Libraries/StellaOps.Attestor.Persistence/StellaOps.Attestor.Persistence.csproj +++ b/src/Attestor/__Libraries/StellaOps.Attestor.Persistence/StellaOps.Attestor.Persistence.csproj @@ -24,6 +24,7 @@ PreserveNewest + Never diff --git a/src/Attestor/__Libraries/StellaOps.Signer.KeyManagement/StellaOps.Signer.KeyManagement.csproj b/src/Attestor/__Libraries/StellaOps.Signer.KeyManagement/StellaOps.Signer.KeyManagement.csproj index 30445e8c0..6085137d5 100644 --- a/src/Attestor/__Libraries/StellaOps.Signer.KeyManagement/StellaOps.Signer.KeyManagement.csproj +++ b/src/Attestor/__Libraries/StellaOps.Signer.KeyManagement/StellaOps.Signer.KeyManagement.csproj @@ -35,6 +35,7 @@ PreserveNewest + Never diff --git a/src/Graph/StellaOps.Graph.Api/Services/GraphDataLoaderHostedService.cs b/src/Graph/StellaOps.Graph.Api/Services/GraphDataLoaderHostedService.cs index 91c9aac48..e49d7e83d 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/GraphDataLoaderHostedService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/GraphDataLoaderHostedService.cs @@ -12,7 +12,7 @@ namespace StellaOps.Graph.Api.Services; internal sealed class GraphDataLoaderHostedService : BackgroundService { private readonly InMemoryGraphRepository _repository; - private readonly PostgresGraphRepository _postgresRepository; + private readonly PostgresGraphRepository? _postgresRepository; private readonly ILogger _logger; private readonly TimeSpan _refreshInterval; @@ -28,7 +28,9 @@ internal sealed class GraphDataLoaderHostedService : BackgroundService ILogger logger) { _repository = repository ?? throw new ArgumentNullException(nameof(repository)); - _postgresRepository = postgresRepository ?? throw new ArgumentNullException(nameof(postgresRepository)); + // postgresRepository may be null when the DI factory returns null! on Postgres + // connection failure (e.g. missing graph schema on fresh volume). + _postgresRepository = postgresRepository; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _refreshInterval = DefaultRefreshInterval; } @@ -62,6 +64,12 @@ internal sealed class GraphDataLoaderHostedService : BackgroundService { try { + if (_postgresRepository is null) + { + _logger.LogInformation("GraphDataLoader: PostgresGraphRepository is unavailable, skipping load"); + return; + } + var available = await _postgresRepository.IsAvailableAsync(ct).ConfigureAwait(false); if (!available) { diff --git a/src/Integrations/__Plugins/StellaOps.Integrations.Plugin.DockerRegistry/DockerRegistryConnectorPlugin.cs b/src/Integrations/__Plugins/StellaOps.Integrations.Plugin.DockerRegistry/DockerRegistryConnectorPlugin.cs index e70a733c6..822ce775e 100644 --- a/src/Integrations/__Plugins/StellaOps.Integrations.Plugin.DockerRegistry/DockerRegistryConnectorPlugin.cs +++ b/src/Integrations/__Plugins/StellaOps.Integrations.Plugin.DockerRegistry/DockerRegistryConnectorPlugin.cs @@ -1,5 +1,6 @@ using StellaOps.Integrations.Contracts; using StellaOps.Integrations.Core; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text.Json; @@ -47,7 +48,7 @@ public sealed class DockerRegistryConnectorPlugin : IIntegrationConnectorPlugin, try { // OCI Distribution Spec: GET /v2/ returns 200 {} when authenticated - var response = await client.GetAsync("/v2/", cancellationToken); + using var response = await SendGetAsync(client, config, "/v2/", cancellationToken); var duration = _timeProvider.GetUtcNow() - startTime; if (response.IsSuccessStatusCode) @@ -99,7 +100,7 @@ public sealed class DockerRegistryConnectorPlugin : IIntegrationConnectorPlugin, try { // Check /v2/_catalog to verify registry is fully operational - var response = await client.GetAsync("/v2/_catalog", cancellationToken); + using var response = await SendGetAsync(client, config, "/v2/_catalog", cancellationToken); var duration = _timeProvider.GetUtcNow() - startTime; if (response.IsSuccessStatusCode) @@ -147,18 +148,19 @@ public sealed class DockerRegistryConnectorPlugin : IIntegrationConnectorPlugin, return IntegrationDiscoveryResourceTypes.Normalize(resourceType) switch { - IntegrationDiscoveryResourceTypes.Repositories => await DiscoverRepositoriesAsync(client, filter, cancellationToken), - IntegrationDiscoveryResourceTypes.Tags => await DiscoverTagsAsync(client, filter, cancellationToken), + IntegrationDiscoveryResourceTypes.Repositories => await DiscoverRepositoriesAsync(client, config, filter, cancellationToken), + IntegrationDiscoveryResourceTypes.Tags => await DiscoverTagsAsync(client, config, filter, cancellationToken), _ => [] }; } private static async Task> DiscoverRepositoriesAsync( HttpClient client, + IntegrationConfig config, IReadOnlyDictionary? filter, CancellationToken cancellationToken) { - var response = await client.GetAsync("/v2/_catalog", cancellationToken); + using var response = await SendGetAsync(client, config, "/v2/_catalog", cancellationToken); response.EnsureSuccessStatusCode(); var content = await response.Content.ReadAsStringAsync(cancellationToken); @@ -179,6 +181,7 @@ public sealed class DockerRegistryConnectorPlugin : IIntegrationConnectorPlugin, private static async Task> DiscoverTagsAsync( HttpClient client, + IntegrationConfig config, IReadOnlyDictionary? filter, CancellationToken cancellationToken) { @@ -193,7 +196,7 @@ public sealed class DockerRegistryConnectorPlugin : IIntegrationConnectorPlugin, repository.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Select(Uri.EscapeDataString)); - var response = await client.GetAsync($"/v2/{repositoryPath}/tags/list", cancellationToken); + using var response = await SendGetAsync(client, config, $"/v2/{repositoryPath}/tags/list", cancellationToken); response.EnsureSuccessStatusCode(); var content = await response.Content.ReadAsStringAsync(cancellationToken); @@ -215,6 +218,125 @@ public sealed class DockerRegistryConnectorPlugin : IIntegrationConnectorPlugin, .ToList(); } + private static async Task SendGetAsync( + HttpClient client, + IntegrationConfig config, + string requestUri, + CancellationToken cancellationToken) + { + var response = await client.GetAsync(requestUri, cancellationToken).ConfigureAwait(false); + if (response.StatusCode != HttpStatusCode.Unauthorized || string.IsNullOrWhiteSpace(config.ResolvedSecret)) + { + return response; + } + + var challenge = response.Headers.WwwAuthenticate + .FirstOrDefault(header => string.Equals(header.Scheme, "Bearer", StringComparison.OrdinalIgnoreCase)); + if (challenge is null || !TryBuildTokenUri(challenge, out var tokenUri)) + { + return response; + } + + var token = await RequestBearerTokenAsync(client, config.ResolvedSecret, tokenUri, cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(token)) + { + return response; + } + + response.Dispose(); + + using var retryRequest = new HttpRequestMessage(HttpMethod.Get, requestUri); + retryRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + retryRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + return await client.SendAsync(retryRequest, cancellationToken).ConfigureAwait(false); + } + + private static async Task RequestBearerTokenAsync( + HttpClient client, + string resolvedSecret, + Uri tokenUri, + CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(HttpMethod.Get, tokenUri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + ApplyResolvedSecret(request.Headers, resolvedSecret); + + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + return null; + } + + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var tokenResponse = JsonSerializer.Deserialize(content, JsonOptions); + return tokenResponse?.Token ?? tokenResponse?.AccessToken; + } + + private static bool TryBuildTokenUri(AuthenticationHeaderValue challenge, out Uri tokenUri) + { + tokenUri = null!; + + var parameters = ParseChallengeParameters(challenge.Parameter); + if (!parameters.TryGetValue("realm", out var realm) || string.IsNullOrWhiteSpace(realm)) + { + return false; + } + + if (!Uri.TryCreate(realm, UriKind.Absolute, out var realmUri)) + { + return false; + } + + var queryParts = new List(); + if (parameters.TryGetValue("service", out var service) && !string.IsNullOrWhiteSpace(service)) + { + queryParts.Add($"service={Uri.EscapeDataString(service)}"); + } + + if (parameters.TryGetValue("scope", out var scope) && !string.IsNullOrWhiteSpace(scope)) + { + queryParts.Add($"scope={Uri.EscapeDataString(scope)}"); + } + + var builder = new UriBuilder(realmUri); + var existingQuery = builder.Query.TrimStart('?'); + builder.Query = string.IsNullOrWhiteSpace(existingQuery) + ? string.Join("&", queryParts) + : queryParts.Count == 0 + ? existingQuery + : $"{existingQuery}&{string.Join("&", queryParts)}"; + + tokenUri = builder.Uri; + return true; + } + + private static Dictionary ParseChallengeParameters(string? parameter) + { + var parameters = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (string.IsNullOrWhiteSpace(parameter)) + { + return parameters; + } + + foreach (var part in parameter.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) + { + var separatorIndex = part.IndexOf('='); + if (separatorIndex <= 0) + { + continue; + } + + var key = part[..separatorIndex].Trim(); + var value = part[(separatorIndex + 1)..].Trim().Trim('"'); + if (!string.IsNullOrWhiteSpace(key)) + { + parameters[key] = value; + } + } + + return parameters; + } + private static HttpClient CreateHttpClient(IntegrationConfig config) { var client = new HttpClient @@ -225,22 +347,29 @@ public sealed class DockerRegistryConnectorPlugin : IIntegrationConnectorPlugin, client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - if (!string.IsNullOrWhiteSpace(config.ResolvedSecret)) - { - if (config.ResolvedSecret.Contains(':', StringComparison.Ordinal)) - { - var credentials = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(config.ResolvedSecret)); - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", credentials); - } - else - { - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", config.ResolvedSecret); - } - } + ApplyResolvedSecret(client.DefaultRequestHeaders, config.ResolvedSecret); return client; } + private static void ApplyResolvedSecret(HttpRequestHeaders headers, string? resolvedSecret) + { + if (string.IsNullOrWhiteSpace(resolvedSecret)) + { + return; + } + + if (resolvedSecret.Contains(':', StringComparison.Ordinal)) + { + var credentials = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(resolvedSecret)); + headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); + } + else + { + headers.Authorization = new AuthenticationHeaderValue("Bearer", resolvedSecret); + } + } + private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true @@ -258,4 +387,10 @@ public sealed class DockerRegistryConnectorPlugin : IIntegrationConnectorPlugin, public string? Name { get; set; } public List? Tags { get; set; } } + + private sealed class RegistryBearerTokenResponse + { + public string? Token { get; set; } + public string? AccessToken { get; set; } + } } diff --git a/src/Integrations/__Tests/StellaOps.Integrations.Plugin.Tests/DockerRegistryConnectorPluginTests.cs b/src/Integrations/__Tests/StellaOps.Integrations.Plugin.Tests/DockerRegistryConnectorPluginTests.cs index 02239f479..685487843 100644 --- a/src/Integrations/__Tests/StellaOps.Integrations.Plugin.Tests/DockerRegistryConnectorPluginTests.cs +++ b/src/Integrations/__Tests/StellaOps.Integrations.Plugin.Tests/DockerRegistryConnectorPluginTests.cs @@ -76,6 +76,33 @@ public sealed class DockerRegistryConnectorPluginTests request.Authorization); } + [Fact] + public async Task TestConnectionAsync_WithBearerChallenge_RetriesWithRegistryToken() + { + LoopbackHttpFixture? fixture = null; + fixture = LoopbackHttpFixture.Start(3, request => request.Path switch + { + "/v2/" when request.Authorization is not null && request.Authorization.StartsWith("Basic ", StringComparison.Ordinal) => + HttpResponse.Challenge($"{fixture!.BaseUrl}/jwt/auth", "container_registry", "registry:catalog:*"), + "/jwt/auth?service=container_registry&scope=registry%3Acatalog%3A%2A" when request.Authorization is not null && request.Authorization.StartsWith("Basic ", StringComparison.Ordinal) => + HttpResponse.Json("""{"token":"registry-jwt"}"""), + "/v2/" when request.Authorization == "Bearer registry-jwt" => + HttpResponse.Json("{}"), + _ => HttpResponse.Text("unexpected-path"), + }); + + var plugin = new DockerRegistryConnectorPlugin(); + var result = await plugin.TestConnectionAsync(CreateConfig(fixture.BaseUrl, "root:registry-pat")); + + var requests = await fixture.WaitForRequestsAsync(); + + Assert.True(result.Success); + Assert.Equal( + ["/v2/", "/jwt/auth?service=container_registry&scope=registry%3Acatalog%3A%2A", "/v2/"], + requests.Select(request => request.Path).ToArray()); + Assert.Equal("Bearer registry-jwt", requests[^1].Authorization); + } + private static IntegrationConfig CreateConfig(string endpoint, string? resolvedSecret = null) { return new IntegrationConfig( @@ -91,23 +118,29 @@ public sealed class DockerRegistryConnectorPluginTests private sealed class LoopbackHttpFixture : IDisposable { private readonly TcpListener _listener; - private readonly Task _requestTask; + private readonly Task> _requestTask; - private LoopbackHttpFixture(Func responder) + private LoopbackHttpFixture(int expectedRequests, Func responder) { _listener = new TcpListener(IPAddress.Loopback, 0); _listener.Start(); BaseUrl = $"http://127.0.0.1:{((IPEndPoint)_listener.LocalEndpoint).Port}"; - _requestTask = HandleSingleRequestAsync(responder); + _requestTask = HandleRequestsAsync(expectedRequests, responder); } public string BaseUrl { get; } - public static LoopbackHttpFixture Start(Func responder) => new(responder); + public static LoopbackHttpFixture Start(Func responder) => + new(1, request => responder(request.Path)); - public async Task WaitForPathAsync() => (await _requestTask).Path; + public static LoopbackHttpFixture Start(int expectedRequests, Func responder) => + new(expectedRequests, responder); - public Task WaitForRequestAsync() => _requestTask; + public async Task WaitForPathAsync() => (await _requestTask)[0].Path; + + public async Task WaitForRequestAsync() => (await _requestTask)[0]; + + public Task> WaitForRequestsAsync() => _requestTask; public void Dispose() { @@ -120,68 +153,96 @@ public sealed class DockerRegistryConnectorPluginTests } } - private async Task HandleSingleRequestAsync(Func responder) + private async Task> HandleRequestsAsync(int expectedRequests, Func responder) { - using var client = await _listener.AcceptTcpClientAsync(); - using var stream = client.GetStream(); - using var reader = new StreamReader( - stream, - Encoding.ASCII, - detectEncodingFromByteOrderMarks: false, - bufferSize: 1024, - leaveOpen: true); + var requests = new List(expectedRequests); - var requestLine = await reader.ReadLineAsync(); - if (string.IsNullOrWhiteSpace(requestLine)) + for (var i = 0; i < expectedRequests; i++) { - throw new InvalidOperationException("Did not receive an HTTP request line."); - } + using var client = await _listener.AcceptTcpClientAsync(); + using var stream = client.GetStream(); + using var reader = new StreamReader( + stream, + Encoding.ASCII, + detectEncodingFromByteOrderMarks: false, + bufferSize: 1024, + leaveOpen: true); - var requestParts = requestLine.Split(' ', StringSplitOptions.RemoveEmptyEntries); - if (requestParts.Length < 2) - { - throw new InvalidOperationException($"Unexpected HTTP request line: {requestLine}"); - } - - string? authorization = null; - while (true) - { - var headerLine = await reader.ReadLineAsync(); - if (string.IsNullOrEmpty(headerLine)) + var requestLine = await reader.ReadLineAsync(); + if (string.IsNullOrWhiteSpace(requestLine)) { - break; + throw new InvalidOperationException("Did not receive an HTTP request line."); } - if (headerLine.StartsWith("Authorization:", StringComparison.OrdinalIgnoreCase)) + var requestParts = requestLine.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (requestParts.Length < 2) { - authorization = headerLine["Authorization:".Length..].Trim(); + throw new InvalidOperationException($"Unexpected HTTP request line: {requestLine}"); } + + string? authorization = null; + while (true) + { + var headerLine = await reader.ReadLineAsync(); + if (string.IsNullOrEmpty(headerLine)) + { + break; + } + + if (headerLine.StartsWith("Authorization:", StringComparison.OrdinalIgnoreCase)) + { + authorization = headerLine["Authorization:".Length..].Trim(); + } + } + + var request = new HttpRequestData(requestParts[1], authorization); + requests.Add(request); + + var response = responder(request); + var payload = Encoding.UTF8.GetBytes(response.Body); + IEnumerable> responseHeaders = response.Headers is null + ? Array.Empty>() + : response.Headers; + var responseText = + $"HTTP/1.1 {response.StatusCode} {response.ReasonPhrase}\r\n" + + $"Content-Type: {response.ContentType}\r\n" + + $"Content-Length: {payload.Length}\r\n" + + string.Concat(responseHeaders.Select(header => $"{header.Key}: {header.Value}\r\n")) + + "Connection: close\r\n" + + "\r\n"; + + var headerBytes = Encoding.ASCII.GetBytes(responseText); + await stream.WriteAsync(headerBytes); + await stream.WriteAsync(payload); + await stream.FlushAsync(); } - var requestPath = requestParts[1]; - var response = responder(requestPath); - var payload = Encoding.UTF8.GetBytes(response.Body); - var responseText = - $"HTTP/1.1 {response.StatusCode} {response.ReasonPhrase}\r\n" + - $"Content-Type: {response.ContentType}\r\n" + - $"Content-Length: {payload.Length}\r\n" + - "Connection: close\r\n" + - "\r\n"; - - var headerBytes = Encoding.ASCII.GetBytes(responseText); - await stream.WriteAsync(headerBytes); - await stream.WriteAsync(payload); - await stream.FlushAsync(); - return new HttpRequestData(requestPath, authorization); + return requests; } } private sealed record HttpRequestData(string Path, string? Authorization); - private sealed record HttpResponse(int StatusCode, string ReasonPhrase, string ContentType, string Body) + private sealed record HttpResponse( + int StatusCode, + string ReasonPhrase, + string ContentType, + string Body, + IReadOnlyDictionary? Headers = null) { public static HttpResponse Json(string body) => new(200, "OK", "application/json", body); public static HttpResponse Text(string body) => new(200, "OK", "text/plain", body); + + public static HttpResponse Challenge(string realm, string service, string scope) => + new( + 401, + "Unauthorized", + "text/plain", + string.Empty, + new Dictionary + { + ["WWW-Authenticate"] = $"Bearer realm=\"{realm}\",service=\"{service}\",scope=\"{scope}\"" + }); } } diff --git a/src/JobEngine/StellaOps.Scheduler.WebService/Program.cs b/src/JobEngine/StellaOps.Scheduler.WebService/Program.cs index 22b741971..322b400ed 100644 --- a/src/JobEngine/StellaOps.Scheduler.WebService/Program.cs +++ b/src/JobEngine/StellaOps.Scheduler.WebService/Program.cs @@ -30,17 +30,11 @@ using StellaOps.Scheduler.WebService.PolicyRuns; using StellaOps.Scheduler.WebService.PolicySimulations; using StellaOps.Scheduler.WebService.Runs; using StellaOps.Scheduler.WebService.Schedules; -using StellaOps.Scheduler.WebService.Scripts; -using StellaOps.Scheduler.WebService.Exceptions; using StellaOps.Scheduler.WebService.VulnerabilityResolverJobs; -using StellaOps.ReleaseOrchestrator.Scripts; -using StellaOps.ReleaseOrchestrator.Scripts.Persistence; -using StellaOps.ReleaseOrchestrator.Scripts.Search; using StellaOps.Scheduler.Worker.Exceptions; using StellaOps.Scheduler.Worker.Observability; using StellaOps.Scheduler.Worker.Options; using StellaOps.Scheduler.Plugin; -using StellaOps.Scheduler.Plugin.Scan; using StellaOps.Scheduler.Plugin.Doctor; using StellaOps.Scheduler.Queue; using StellaOps.Scheduler.Worker.DependencyInjection; @@ -129,16 +123,6 @@ else builder.Services.AddSingleton(); builder.Services.AddSingleton(); } -// Scripts registry (shares the same Postgres options as Scheduler) -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); - // Workflow engine HTTP client (starts workflow instances for system schedules) builder.Services.AddHttpClient((sp, client) => { @@ -219,7 +203,7 @@ builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions var pluginRegistry = new SchedulerPluginRegistry(); // Built-in: ScanJobPlugin (handles jobKind="scan") -var scanPlugin = new ScanJobPlugin(); +var scanPlugin = new StellaOps.Scheduler.Plugin.Scan.ScanJobPlugin(); pluginRegistry.Register(scanPlugin); // Built-in: DoctorJobPlugin (handles jobKind="doctor") @@ -385,7 +369,6 @@ app.MapFailureSignatureEndpoints(); app.MapPolicyRunEndpoints(); app.MapPolicySimulationEndpoints(); app.MapSchedulerEventWebhookEndpoints(); -app.MapScriptsEndpoints(); // Map plugin-registered endpoints (e.g. Doctor trend endpoints) var registry = app.Services.GetRequiredService(); diff --git a/src/JobEngine/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj b/src/JobEngine/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj index 44512782c..a0474ec78 100644 --- a/src/JobEngine/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj +++ b/src/JobEngine/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj @@ -27,7 +27,6 @@ - diff --git a/src/Timeline/StellaOps.Timeline.WebService/Program.cs b/src/Timeline/StellaOps.Timeline.WebService/Program.cs index 504af41dc..0ae722efd 100644 --- a/src/Timeline/StellaOps.Timeline.WebService/Program.cs +++ b/src/Timeline/StellaOps.Timeline.WebService/Program.cs @@ -1,4 +1,5 @@ using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Hosting; using StellaOps.Auth.Abstractions; using StellaOps.Cryptography.Audit; using StellaOps.Infrastructure.Postgres.Migrations; @@ -15,6 +16,7 @@ using StellaOps.Timeline.WebService.Endpoints; using StellaOps.Timeline.WebService.Security; using StellaOps.TimelineIndexer.Core.Abstractions; using StellaOps.TimelineIndexer.Core.Models; +using StellaOps.TimelineIndexer.Infrastructure; using StellaOps.TimelineIndexer.Infrastructure.DependencyInjection; using StellaOps.TimelineIndexer.Infrastructure.Options; using StellaOps.TimelineIndexer.Infrastructure.Subscriptions; @@ -84,6 +86,15 @@ builder.Services.AddStartupMigrations( migrationsAssembly: typeof(TimelineCoreDataSource).Assembly, connectionStringSelector: opts => opts.ConnectionString); +// Auto-migrate: timeline indexer schema (events, details, digests tables) +// Uses AddStartupMigrations instead of the library's built-in TimelineIndexerMigrationHostedService +// to avoid two competing migration runners targeting the same 'timeline' schema. +builder.Services.AddStartupMigrations( + schemaName: TimelineIndexerDataSource.DefaultSchemaName, + moduleName: "TimelineIndexer", + migrationsAssembly: typeof(TimelineIndexerDataSource).Assembly, + connectionStringSelector: opts => opts.ConnectionString); + // Audit event providers: Postgres persistence (primary) + HTTP polling (transitional fallback) builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -92,6 +103,18 @@ builder.Services.AddSingleton= 0; i--) +{ + var sd = builder.Services[i]; + if (sd.ServiceType == typeof(IHostedService) + && sd.ImplementationType?.Name == "TimelineIndexerMigrationHostedService") + { + builder.Services.RemoveAt(i); + } +} builder.Services.AddSingleton(); // ── Timeline Indexer ingestion worker (merged from timeline-indexer-worker) ──