diff --git a/docs/modules/release-orchestrator/architecture.md b/docs/modules/release-orchestrator/architecture.md index 300509149..bab339cce 100644 --- a/docs/modules/release-orchestrator/architecture.md +++ b/docs/modules/release-orchestrator/architecture.md @@ -4,7 +4,7 @@ **Status:** Active Development (backend substantially implemented; API surface layer in progress) -> **Implementation reality (updated 2026-03-31):** The backend is substantially complete with 140,000+ lines of production code across 49 projects. Core libraries (Release, Promotion, Deployment, Workflow, Evidence, PolicyGate, Progressive, Federation, Compliance) are implemented with comprehensive tests (283 test files, 37K lines). Six agent types are operational (Compose, Docker, SSH, WinRM, ECS, Nomad). Compatibility HTTP surfaces now exist across Platform, JobEngine, and Scanner for environment management, deployment monitoring, evidence inspection, dashboard promotion decisions, and registry search. **Remaining gaps:** the dedicated Release Orchestrator WebApi host is still incomplete, storage remains in-memory for these compatibility surfaces, and first-class migrations/persistence for the standalone API are still pending. +> **Implementation reality (updated 2026-04-10):** The backend is substantially complete with 140,000+ lines of production code across 49 projects. Core libraries (Release, Promotion, Deployment, Workflow, Evidence, PolicyGate, Progressive, Federation, Compliance) are implemented with comprehensive tests (283 test files, 37K lines). Six agent types are operational (Compose, Docker, SSH, WinRM, ECS, Nomad). Compatibility HTTP surfaces now exist across Platform, JobEngine, and Scanner for environment management, deployment monitoring, evidence inspection, dashboard promotion decisions, and registry search. The standalone WebApi now owns and auto-migrates the `scripts` PostgreSQL schema used by `/api/v2/scripts`, so fresh local installs no longer depend on Scheduler-owned SQL bootstrap for the scripts catalog. **Remaining gaps:** the dedicated Release Orchestrator WebApi host is still incomplete, and many compatibility surfaces still rely on in-memory storage outside the scripts catalog and audit/first-signal persistence. ## Overview diff --git a/src/ReleaseOrchestrator/__Apps/StellaOps.ReleaseOrchestrator.WebApi/Program.cs b/src/ReleaseOrchestrator/__Apps/StellaOps.ReleaseOrchestrator.WebApi/Program.cs index f3a23a8c2..b3329e405 100644 --- a/src/ReleaseOrchestrator/__Apps/StellaOps.ReleaseOrchestrator.WebApi/Program.cs +++ b/src/ReleaseOrchestrator/__Apps/StellaOps.ReleaseOrchestrator.WebApi/Program.cs @@ -2,11 +2,9 @@ using StellaOps.Audit.Emission; using StellaOps.Router.AspNet; using StellaOps.Auth.ServerIntegration; using StellaOps.Auth.ServerIntegration.Tenancy; -using StellaOps.Infrastructure.Postgres.Options; using StellaOps.ReleaseOrchestrator.Persistence.Extensions; using StellaOps.ReleaseOrchestrator.Scripts; using StellaOps.ReleaseOrchestrator.Scripts.Persistence; -using StellaOps.ReleaseOrchestrator.Scripts.Search; using StellaOps.ReleaseOrchestrator.WebApi; using StellaOps.ReleaseOrchestrator.WebApi.Endpoints; using StellaOps.ReleaseOrchestrator.WebApi.Services; @@ -47,29 +45,27 @@ builder.Services.AddSingleton(sp => // Scripts registry (owns the 'scripts' schema — moved from scheduler to fix dual-schema violation) var scriptsSection = builder.Configuration.GetSection("Scripts:Postgres"); -if (scriptsSection.Exists()) -{ - builder.Services.Configure(scriptsSection); -} -else -{ - // Fallback: reuse the default connection string with scripts schema - builder.Services.Configure(opt => +builder.Services.AddOptions() + .Configure(options => { - opt.ConnectionString = builder.Configuration.GetConnectionString("Default") - ?? builder.Configuration["ConnectionStrings__Default"] - ?? "Host=localhost;Database=stellaops_platform;Username=stellaops;Password=stellaops"; - opt.SchemaName = "scripts"; + if (scriptsSection.Exists()) + { + scriptsSection.Bind(options); + } + + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + options.ConnectionString = builder.Configuration.GetConnectionString("Default") + ?? builder.Configuration["ConnectionStrings__Default"] + ?? "Host=localhost;Database=stellaops_platform;Username=stellaops;Password=stellaops"; + } + + if (string.IsNullOrWhiteSpace(options.SchemaName)) + { + options.SchemaName = ScriptsDataSource.DefaultSchemaName; + } }); -} -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); -builder.Services.AddSingleton(); +builder.Services.AddReleaseOrchestratorScripts(); // Unified audit emission (posts audit events to Timeline service) builder.Services.AddAuditEmission(builder.Configuration); diff --git a/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Migrations/001_initial.sql b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Migrations/001_initial.sql new file mode 100644 index 000000000..7880519fa --- /dev/null +++ b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Migrations/001_initial.sql @@ -0,0 +1,136 @@ +-- 001_initial.sql +-- Creates the scripts schema and seeds the built-in sample catalog used by the UI. + +CREATE SCHEMA IF NOT EXISTS scripts; + +CREATE TABLE IF NOT EXISTS scripts.scripts ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + language TEXT NOT NULL, + content TEXT NOT NULL, + entry_point TEXT, + version INT NOT NULL DEFAULT 1, + dependencies JSONB NOT NULL DEFAULT '[]', + tags TEXT[] NOT NULL DEFAULT '{}', + variables JSONB NOT NULL DEFAULT '[]', + visibility TEXT NOT NULL DEFAULT 'private', + owner_id TEXT NOT NULL, + team_id TEXT, + content_hash TEXT NOT NULL, + is_sample BOOLEAN NOT NULL DEFAULT FALSE, + sample_category TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_scripts_owner ON scripts.scripts (owner_id); +CREATE INDEX IF NOT EXISTS idx_scripts_lang ON scripts.scripts (language); +CREATE INDEX IF NOT EXISTS idx_scripts_vis ON scripts.scripts (visibility); +CREATE INDEX IF NOT EXISTS idx_scripts_sample ON scripts.scripts (is_sample) WHERE is_sample = TRUE; +CREATE INDEX IF NOT EXISTS idx_scripts_updated ON scripts.scripts (updated_at DESC NULLS LAST, created_at DESC); + +CREATE TABLE IF NOT EXISTS scripts.script_versions ( + script_id TEXT NOT NULL REFERENCES scripts.scripts(id) ON DELETE CASCADE, + version INT NOT NULL, + content TEXT NOT NULL, + content_hash TEXT NOT NULL, + dependencies JSONB NOT NULL DEFAULT '[]', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by TEXT NOT NULL, + change_note TEXT, + PRIMARY KEY (script_id, version) +); + +INSERT INTO scripts.scripts ( + id, + name, + description, + language, + content, + version, + tags, + variables, + visibility, + owner_id, + content_hash, + is_sample, + sample_category, + created_at, + updated_at) +VALUES +( + 'scr-001', + 'Pre-deploy Health Check', + 'Validates service health endpoints before deployment proceeds. Checks HTTP status, response time, and dependency connectivity.', + 'bash', + E'#!/bin/bash\n# Pre-deploy health check script\nset -euo pipefail\n\nSERVICE_URL="${SERVICE_URL:-http://localhost:8080}"\nTIMEOUT=${TIMEOUT:-10}\n\necho "Checking health at $SERVICE_URL/health..."\nHTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time "$TIMEOUT" "$SERVICE_URL/health")\n\nif [ "$HTTP_CODE" -eq 200 ]; then\n echo "Health check passed (HTTP $HTTP_CODE)"\n exit 0\nelse\n echo "Health check failed (HTTP $HTTP_CODE)"\n exit 1\nfi', + 3, + ARRAY['health-check', 'pre-deploy', 'infrastructure'], + '[{"name":"SERVICE_URL","description":"Target service URL for health check","isRequired":true,"defaultValue":"http://localhost:8080","isSecret":false},{"name":"TIMEOUT","description":"Request timeout in seconds","isRequired":false,"defaultValue":"10","isSecret":false}]'::jsonb, + 'organization', + 'admin', + 'sha256:a1b2c3d4e5f6', + TRUE, + 'deployment', + '2026-01-10T08:00:00Z', + '2026-03-15T14:30:00Z' +), +( + 'scr-002', + 'Database Migration Validator', + 'Validates pending database migrations against schema constraints and checks for backward compatibility.', + 'python', + E'"""Database migration validator."""\nimport sys\nimport hashlib\n\ndef validate_migration(migration_path: str) -> bool:\n """Validate a single migration file."""\n with open(migration_path, ''r'') as f:\n content = f.read()\n\n destructive_ops = [''DROP TABLE'', ''DROP COLUMN'', ''TRUNCATE'']\n for op in destructive_ops:\n if op in content.upper():\n print(f"WARNING: Destructive operation found: {op}")\n return False\n\n checksum = hashlib.sha256(content.encode()).hexdigest()\n print(f"Migration checksum: {checksum[:16]}")\n return True\n\nif __name__ == ''__main__'':\n path = sys.argv[1] if len(sys.argv) > 1 else ''migrations/''\n result = validate_migration(path)\n sys.exit(0 if result else 1)', + 2, + ARRAY['database', 'migration', 'validation'], + '[{"name":"DB_CONNECTION","description":"Database connection string","isRequired":true,"isSecret":true},{"name":"MIGRATION_DIR","description":"Path to migrations directory","isRequired":false,"defaultValue":"migrations/","isSecret":false}]'::jsonb, + 'team', + 'admin', + 'sha256:b2c3d4e5f6a7', + TRUE, + 'database', + '2026-02-01T10:00:00Z', + '2026-03-10T09:15:00Z' +), +( + 'scr-003', + 'Release Notes Generator', + 'Generates release notes from git commit history between two tags, grouped by conventional commit type.', + 'typescript', + E'/**\n * Release notes generator.\n * Parses conventional commits and groups them by type.\n */\ninterface CommitEntry {\n hash: string;\n type: string;\n scope?: string;\n message: string;\n}\n\nfunction parseConventionalCommit(line: string): CommitEntry | null {\n const match = line.match(/^(\\w+)(\\((\\w+)\\))?:\\s+(.+)$/);\n if (!match) return null;\n return { hash: '''', type: match[1], scope: match[3], message: match[4] };\n}\n\nconsole.log(''Release notes generator ready.'');', + 1, + ARRAY['release-notes', 'git', 'automation'], + '[]'::jsonb, + 'public', + 'admin', + 'sha256:c3d4e5f6a7b8', + TRUE, + 'release', + '2026-03-01T12:00:00Z', + '2026-03-01T12:00:00Z' +), +( + 'scr-004', + 'Container Image Scan Wrapper', + 'Wraps Trivy container scanning with custom policy checks and outputs results in SARIF format.', + 'csharp', + E'// Container image scan wrapper\nusing System;\nusing System.Diagnostics;\nusing System.Text.Json;\n\nvar imageRef = Environment.GetEnvironmentVariable("IMAGE_REF")\n ?? throw new InvalidOperationException("IMAGE_REF not set");\n\nvar severityThreshold = Environment.GetEnvironmentVariable("SEVERITY_THRESHOLD") ?? "HIGH";\n\nConsole.WriteLine($"Scanning {imageRef} with threshold {severityThreshold}...");', + 5, + ARRAY['security', 'scanning', 'trivy', 'container'], + '[{"name":"IMAGE_REF","description":"Container image reference to scan","isRequired":true,"isSecret":false},{"name":"SEVERITY_THRESHOLD","description":"Minimum severity to report","isRequired":false,"defaultValue":"HIGH","isSecret":false}]'::jsonb, + 'organization', + 'admin', + 'sha256:d4e5f6a7b8c9', + FALSE, + NULL, + '2026-01-20T16:00:00Z', + '2026-03-20T11:45:00Z' +) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO scripts.script_versions (script_id, version, content, content_hash, dependencies, created_at, created_by, change_note) +SELECT id, version, content, content_hash, dependencies, created_at, owner_id, 'Current version' +FROM scripts.scripts +WHERE id IN ('scr-001', 'scr-002', 'scr-003', 'scr-004') +ON CONFLICT (script_id, version) DO NOTHING; diff --git a/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Persistence/ScriptsDataSource.cs b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Persistence/ScriptsDataSource.cs index 2c4af8769..5b929b26c 100644 --- a/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Persistence/ScriptsDataSource.cs +++ b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Persistence/ScriptsDataSource.cs @@ -12,19 +12,15 @@ public sealed class ScriptsDataSource : DataSourceBase { public const string DefaultSchemaName = "scripts"; - public ScriptsDataSource(IOptions options, ILogger logger) + public ScriptsDataSource(IOptions options, ILogger logger) : base(CreateOptions(options.Value), logger) { } protected override string ModuleName => "Scripts"; - private static PostgresOptions CreateOptions(PostgresOptions baseOptions) + private static PostgresOptions CreateOptions(ScriptsPostgresOptions baseOptions) { - if (string.IsNullOrWhiteSpace(baseOptions.SchemaName)) - { - baseOptions.SchemaName = DefaultSchemaName; - } - return baseOptions; + return baseOptions.ToPostgresOptions(); } } diff --git a/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Persistence/ScriptsPostgresOptions.cs b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Persistence/ScriptsPostgresOptions.cs new file mode 100644 index 000000000..2529046bd --- /dev/null +++ b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/Persistence/ScriptsPostgresOptions.cs @@ -0,0 +1,48 @@ +using StellaOps.Infrastructure.Postgres.Options; + +namespace StellaOps.ReleaseOrchestrator.Scripts.Persistence; + +/// +/// Dedicated PostgreSQL options for the scripts catalog. +/// +public sealed class ScriptsPostgresOptions +{ + public string ConnectionString { get; set; } = string.Empty; + + public int CommandTimeoutSeconds { get; set; } = 30; + + public int MaxPoolSize { get; set; } = 100; + + public string? ApplicationName { get; set; } + + public int MinPoolSize { get; set; } = 1; + + public int ConnectionIdleLifetimeSeconds { get; set; } = 300; + + public bool Pooling { get; set; } = true; + + public string? SchemaName { get; set; } + + public bool AutoMigrate { get; set; } + + public string? MigrationsPath { get; set; } + + public PostgresOptions ToPostgresOptions() + { + return new PostgresOptions + { + ConnectionString = ConnectionString, + CommandTimeoutSeconds = CommandTimeoutSeconds, + MaxPoolSize = MaxPoolSize, + ApplicationName = ApplicationName, + MinPoolSize = MinPoolSize, + ConnectionIdleLifetimeSeconds = ConnectionIdleLifetimeSeconds, + Pooling = Pooling, + SchemaName = string.IsNullOrWhiteSpace(SchemaName) + ? ScriptsDataSource.DefaultSchemaName + : SchemaName, + AutoMigrate = AutoMigrate, + MigrationsPath = MigrationsPath + }; + } +} diff --git a/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/ServiceCollectionExtensions.cs b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/ServiceCollectionExtensions.cs new file mode 100644 index 000000000..3b5581fb7 --- /dev/null +++ b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/ServiceCollectionExtensions.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Infrastructure.Postgres.Migrations; +using StellaOps.ReleaseOrchestrator.Scripts.Persistence; +using StellaOps.ReleaseOrchestrator.Scripts.Search; + +namespace StellaOps.ReleaseOrchestrator.Scripts; + +/// +/// Registers the scripts catalog services owned by Release Orchestrator. +/// +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddReleaseOrchestratorScripts(this IServiceCollection services) + { + services.AddStartupMigrations( + schemaName: ScriptsDataSource.DefaultSchemaName, + moduleName: "ReleaseOrchestrator.Scripts", + migrationsAssembly: typeof(ScriptsDataSource).Assembly, + connectionStringSelector: options => options.ConnectionString); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/StellaOps.ReleaseOrchestrator.Scripts.csproj b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/StellaOps.ReleaseOrchestrator.Scripts.csproj index d2c398354..7335fc244 100644 --- a/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/StellaOps.ReleaseOrchestrator.Scripts.csproj +++ b/src/ReleaseOrchestrator/__Libraries/StellaOps.ReleaseOrchestrator.Scripts/StellaOps.ReleaseOrchestrator.Scripts.csproj @@ -10,6 +10,9 @@ + + + @@ -37,4 +40,8 @@ + + + + diff --git a/src/ReleaseOrchestrator/__Tests/StellaOps.ReleaseOrchestrator.Integration.Tests/ScriptsInfrastructureRegistrationTests.cs b/src/ReleaseOrchestrator/__Tests/StellaOps.ReleaseOrchestrator.Integration.Tests/ScriptsInfrastructureRegistrationTests.cs new file mode 100644 index 000000000..d3d2f7db8 --- /dev/null +++ b/src/ReleaseOrchestrator/__Tests/StellaOps.ReleaseOrchestrator.Integration.Tests/ScriptsInfrastructureRegistrationTests.cs @@ -0,0 +1,75 @@ +using FluentAssertions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using StellaOps.ReleaseOrchestrator.Persistence.Extensions; +using StellaOps.ReleaseOrchestrator.Scripts; +using StellaOps.ReleaseOrchestrator.Scripts.Persistence; + +namespace StellaOps.ReleaseOrchestrator.Integration.Tests; + +public sealed class ScriptsInfrastructureRegistrationTests +{ + [Fact] + public void AddReleaseOrchestratorScripts_RegistersStartupMigrationHostAndCoreServices() + { + var services = new ServiceCollection(); + services.AddLogging(); + services.AddOptions() + .Configure(options => + { + options.ConnectionString = "Host=postgres;Database=stellaops;Username=postgres;Password=postgres"; + options.SchemaName = ScriptsDataSource.DefaultSchemaName; + }); + + services.AddReleaseOrchestratorScripts(); + + services + .Where(descriptor => descriptor.ServiceType == typeof(IHostedService)) + .Should() + .ContainSingle("fresh installs need the scripts schema to auto-migrate before /api/v2/scripts can query it"); + + services.Should().Contain(descriptor => descriptor.ServiceType == typeof(ScriptsDataSource)); + services.Should().Contain(descriptor => descriptor.ServiceType == typeof(IScriptStore)); + services.Should().Contain(descriptor => descriptor.ServiceType == typeof(IScriptRegistry)); + } + + [Fact] + public void ScriptsAssembly_EmbedsInitialScriptsSchemaMigration() + { + var resourceNames = typeof(ScriptsDataSource).Assembly.GetManifestResourceNames(); + + resourceNames.Should().Contain( + name => name.EndsWith("Migrations.001_initial.sql", StringComparison.Ordinal), + "the service that serves /api/v2/scripts must carry its own scripts schema DDL for fresh local installs"); + } + + [Fact] + public void ReleaseOrchestratorAndScriptsMigrations_CoexistInOneServiceCollection() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["ReleaseOrchestrator:Postgres:ConnectionString"] = "Host=postgres;Database=stellaops;Username=postgres;Password=postgres", + ["Scripts:Postgres:ConnectionString"] = "Host=postgres;Database=stellaops;Username=postgres;Password=postgres", + ["Scripts:Postgres:SchemaName"] = ScriptsDataSource.DefaultSchemaName + }) + .Build(); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddReleaseOrchestratorPersistence(configuration); + services.AddOptions() + .Configure(options => + { + options.ConnectionString = configuration["Scripts:Postgres:ConnectionString"]!; + options.SchemaName = ScriptsDataSource.DefaultSchemaName; + }); + services.AddReleaseOrchestratorScripts(); + + services + .Where(descriptor => descriptor.ServiceType == typeof(IHostedService)) + .Should() + .HaveCount(2, "Release Orchestrator and scripts each own a separate PostgreSQL schema that must auto-migrate on startup"); + } +} diff --git a/src/ReleaseOrchestrator/__Tests/StellaOps.ReleaseOrchestrator.Integration.Tests/StellaOps.ReleaseOrchestrator.Integration.Tests.csproj b/src/ReleaseOrchestrator/__Tests/StellaOps.ReleaseOrchestrator.Integration.Tests/StellaOps.ReleaseOrchestrator.Integration.Tests.csproj index 91c489b9c..0a5395704 100644 --- a/src/ReleaseOrchestrator/__Tests/StellaOps.ReleaseOrchestrator.Integration.Tests/StellaOps.ReleaseOrchestrator.Integration.Tests.csproj +++ b/src/ReleaseOrchestrator/__Tests/StellaOps.ReleaseOrchestrator.Integration.Tests/StellaOps.ReleaseOrchestrator.Integration.Tests.csproj @@ -11,6 +11,7 @@ + @@ -22,6 +23,8 @@ + + diff --git a/src/__Libraries/StellaOps.Infrastructure.Postgres/Migrations/MigrationServiceExtensions.cs b/src/__Libraries/StellaOps.Infrastructure.Postgres/Migrations/MigrationServiceExtensions.cs index b4ad26591..bbc14830f 100644 --- a/src/__Libraries/StellaOps.Infrastructure.Postgres/Migrations/MigrationServiceExtensions.cs +++ b/src/__Libraries/StellaOps.Infrastructure.Postgres/Migrations/MigrationServiceExtensions.cs @@ -36,7 +36,9 @@ public static class MigrationServiceExtensions var migrationOptions = new StartupMigrationOptions(); configureOptions?.Invoke(migrationOptions); - services.AddHostedService(sp => + // Multiple modules can own distinct schemas inside one service host. + // Register each migration host explicitly so they do not get deduplicated. + services.AddSingleton(sp => { var options = sp.GetRequiredService>().Value; var connectionString = connectionStringSelector(options);