Fix build and code structure improvements. New but essential UI functionality. CI improvements. Documentation improvements. AI module improvements.

This commit is contained in:
StellaOps Bot
2025-12-26 21:54:17 +02:00
parent 335ff7da16
commit c2b9cd8d1f
3717 changed files with 264714 additions and 48202 deletions

View File

@@ -0,0 +1,130 @@
# =============================================================================
# LOCAL CI TESTING SERVICES
# =============================================================================
# Docker Compose profile for running CI tests locally.
# Uses different ports to avoid conflicts with development services.
#
# Usage:
# docker compose -f devops/compose/docker-compose.ci.yaml up -d
# docker compose -f devops/compose/docker-compose.ci.yaml down -v
#
# Services:
# - postgres-ci: PostgreSQL 16 for integration tests (port 5433)
# - valkey-ci: Valkey/Redis for caching tests (port 6380)
# - nats-ci: NATS JetStream for messaging tests (port 4223)
# - mock-registry: Local container registry for release testing (port 5001)
#
# =============================================================================
networks:
ci-net:
driver: bridge
name: stellaops-ci-net
volumes:
ci-postgres-data:
name: stellaops-ci-postgres
ci-valkey-data:
name: stellaops-ci-valkey
services:
# ---------------------------------------------------------------------------
# PostgreSQL 16 - Primary database for integration tests
# ---------------------------------------------------------------------------
postgres-ci:
image: postgres:16-alpine
container_name: stellaops-postgres-ci
environment:
POSTGRES_USER: stellaops_ci
POSTGRES_PASSWORD: ci_test_password
POSTGRES_DB: stellaops_test
# Performance tuning for tests
POSTGRES_INITDB_ARGS: "--data-checksums"
ports:
- "5433:5432" # Different port to avoid conflicts with dev
volumes:
- ci-postgres-data:/var/lib/postgresql/data
networks:
- ci-net
healthcheck:
test: ["CMD-SHELL", "pg_isready -U stellaops_ci -d stellaops_test"]
interval: 5s
timeout: 5s
retries: 10
start_period: 10s
restart: unless-stopped
# ---------------------------------------------------------------------------
# Valkey 8.0 - Redis-compatible cache for caching tests
# ---------------------------------------------------------------------------
valkey-ci:
image: valkey/valkey:8.0-alpine
container_name: stellaops-valkey-ci
command: ["valkey-server", "--appendonly", "yes", "--maxmemory", "256mb", "--maxmemory-policy", "allkeys-lru"]
ports:
- "6380:6379" # Different port to avoid conflicts
volumes:
- ci-valkey-data:/data
networks:
- ci-net
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 5s
retries: 5
restart: unless-stopped
# ---------------------------------------------------------------------------
# NATS JetStream - Message queue for messaging tests
# ---------------------------------------------------------------------------
nats-ci:
image: nats:2.10-alpine
container_name: stellaops-nats-ci
command: ["-js", "-sd", "/data", "-m", "8222"]
ports:
- "4223:4222" # Client port (different from dev)
- "8223:8222" # Monitoring port
networks:
- ci-net
healthcheck:
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8222/healthz"]
interval: 5s
timeout: 5s
retries: 5
restart: unless-stopped
# ---------------------------------------------------------------------------
# Mock Container Registry - For release dry-run testing
# ---------------------------------------------------------------------------
mock-registry:
image: registry:2
container_name: stellaops-registry-ci
ports:
- "5001:5000"
environment:
REGISTRY_STORAGE_DELETE_ENABLED: "true"
networks:
- ci-net
restart: unless-stopped
# ---------------------------------------------------------------------------
# Mock S3 (MinIO) - For artifact storage tests
# ---------------------------------------------------------------------------
minio-ci:
image: minio/minio:latest
container_name: stellaops-minio-ci
command: server /data --console-address ":9001"
ports:
- "9100:9000" # S3 API port
- "9101:9001" # Console port
environment:
MINIO_ROOT_USER: minioadmin
MINIO_ROOT_PASSWORD: minioadmin
networks:
- ci-net
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 10s
timeout: 5s
retries: 5
restart: unless-stopped

View File

@@ -28,6 +28,7 @@ services:
PGDATA: /var/lib/postgresql/data/pgdata
volumes:
- postgres-data:/var/lib/postgresql/data
- ./postgres-init:/docker-entrypoint-initdb.d:ro
ports:
- "${POSTGRES_PORT:-5432}:5432"
networks:

View File

@@ -1,5 +1,7 @@
-- PostgreSQL initialization for StellaOps air-gap deployment
-- ============================================================================
-- PostgreSQL initialization for StellaOps
-- This script runs automatically on first container start
-- ============================================================================
-- Enable pg_stat_statements extension for query performance analysis
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
@@ -9,25 +11,59 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm; -- Fuzzy text search
CREATE EXTENSION IF NOT EXISTS btree_gin; -- GIN indexes for scalar types
CREATE EXTENSION IF NOT EXISTS pgcrypto; -- Cryptographic functions
-- ============================================================================
-- Create schemas for all modules
-- Migrations will create tables within these schemas
CREATE SCHEMA IF NOT EXISTS authority;
CREATE SCHEMA IF NOT EXISTS vuln;
CREATE SCHEMA IF NOT EXISTS vex;
CREATE SCHEMA IF NOT EXISTS scheduler;
CREATE SCHEMA IF NOT EXISTS notify;
CREATE SCHEMA IF NOT EXISTS policy;
CREATE SCHEMA IF NOT EXISTS concelier;
CREATE SCHEMA IF NOT EXISTS audit;
CREATE SCHEMA IF NOT EXISTS unknowns;
-- ============================================================================
-- Grant usage to application user (assumes POSTGRES_USER is the app user)
GRANT USAGE ON SCHEMA authority TO PUBLIC;
GRANT USAGE ON SCHEMA vuln TO PUBLIC;
GRANT USAGE ON SCHEMA vex TO PUBLIC;
GRANT USAGE ON SCHEMA scheduler TO PUBLIC;
GRANT USAGE ON SCHEMA notify TO PUBLIC;
GRANT USAGE ON SCHEMA policy TO PUBLIC;
GRANT USAGE ON SCHEMA concelier TO PUBLIC;
GRANT USAGE ON SCHEMA audit TO PUBLIC;
GRANT USAGE ON SCHEMA unknowns TO PUBLIC;
-- Core Platform
CREATE SCHEMA IF NOT EXISTS authority; -- Authentication, authorization, OAuth/OIDC
-- Data Ingestion
CREATE SCHEMA IF NOT EXISTS vuln; -- Concelier vulnerability data
CREATE SCHEMA IF NOT EXISTS vex; -- Excititor VEX documents
-- Scanning & Analysis
CREATE SCHEMA IF NOT EXISTS scanner; -- Container scanning, SBOM generation
-- Scheduling & Orchestration
CREATE SCHEMA IF NOT EXISTS scheduler; -- Job scheduling
CREATE SCHEMA IF NOT EXISTS taskrunner; -- Task execution
-- Policy & Risk
CREATE SCHEMA IF NOT EXISTS policy; -- Policy engine
CREATE SCHEMA IF NOT EXISTS unknowns; -- Unknown component tracking
-- Artifacts & Evidence
CREATE SCHEMA IF NOT EXISTS proofchain; -- Attestor proof chains
CREATE SCHEMA IF NOT EXISTS attestor; -- Attestor submission queue
CREATE SCHEMA IF NOT EXISTS signer; -- Key management
-- Notifications
CREATE SCHEMA IF NOT EXISTS notify; -- Notification delivery
-- Signals & Observability
CREATE SCHEMA IF NOT EXISTS signals; -- Runtime signals
-- Registry
CREATE SCHEMA IF NOT EXISTS packs; -- Task packs registry
-- Audit
CREATE SCHEMA IF NOT EXISTS audit; -- System-wide audit log
-- ============================================================================
-- Grant usage to application user (for single-user mode)
-- Per-module users are created in 02-create-users.sql
-- ============================================================================
DO $$
DECLARE
schema_name TEXT;
BEGIN
FOR schema_name IN SELECT unnest(ARRAY[
'authority', 'vuln', 'vex', 'scanner', 'scheduler', 'taskrunner',
'policy', 'unknowns', 'proofchain', 'attestor', 'signer',
'notify', 'signals', 'packs', 'audit'
]) LOOP
EXECUTE format('GRANT USAGE ON SCHEMA %I TO PUBLIC', schema_name);
END LOOP;
END $$;

View File

@@ -0,0 +1,53 @@
-- ============================================================================
-- Per-Module Database Users
-- ============================================================================
-- Creates isolated database users for each StellaOps module.
-- This enables least-privilege access control and audit trail per module.
--
-- Password format: {module}_dev (for development only)
-- In production, use secrets management and rotate credentials.
-- ============================================================================
-- Core Platform
CREATE USER authority_user WITH PASSWORD 'authority_dev';
-- Data Ingestion
CREATE USER concelier_user WITH PASSWORD 'concelier_dev';
CREATE USER excititor_user WITH PASSWORD 'excititor_dev';
-- Scanning & Analysis
CREATE USER scanner_user WITH PASSWORD 'scanner_dev';
-- Scheduling & Orchestration
CREATE USER scheduler_user WITH PASSWORD 'scheduler_dev';
CREATE USER taskrunner_user WITH PASSWORD 'taskrunner_dev';
-- Policy & Risk
CREATE USER policy_user WITH PASSWORD 'policy_dev';
CREATE USER unknowns_user WITH PASSWORD 'unknowns_dev';
-- Artifacts & Evidence
CREATE USER attestor_user WITH PASSWORD 'attestor_dev';
CREATE USER signer_user WITH PASSWORD 'signer_dev';
-- Notifications
CREATE USER notify_user WITH PASSWORD 'notify_dev';
-- Signals & Observability
CREATE USER signals_user WITH PASSWORD 'signals_dev';
-- Registry
CREATE USER packs_user WITH PASSWORD 'packs_dev';
-- ============================================================================
-- Log created users
-- ============================================================================
DO $$
BEGIN
RAISE NOTICE 'Created per-module database users:';
RAISE NOTICE ' - authority_user, concelier_user, excititor_user';
RAISE NOTICE ' - scanner_user, scheduler_user, taskrunner_user';
RAISE NOTICE ' - policy_user, unknowns_user';
RAISE NOTICE ' - attestor_user, signer_user';
RAISE NOTICE ' - notify_user, signals_user, packs_user';
END $$;

View File

@@ -0,0 +1,153 @@
-- ============================================================================
-- Per-Module Schema Permissions
-- ============================================================================
-- Grants each module user access to their respective schema(s).
-- Users can only access tables in their designated schemas.
-- ============================================================================
-- ============================================================================
-- Authority Module
-- ============================================================================
GRANT USAGE ON SCHEMA authority TO authority_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA authority TO authority_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA authority TO authority_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA authority GRANT ALL ON TABLES TO authority_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA authority GRANT ALL ON SEQUENCES TO authority_user;
-- ============================================================================
-- Concelier Module (uses 'vuln' schema)
-- ============================================================================
GRANT USAGE ON SCHEMA vuln TO concelier_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA vuln TO concelier_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA vuln TO concelier_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA vuln GRANT ALL ON TABLES TO concelier_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA vuln GRANT ALL ON SEQUENCES TO concelier_user;
-- ============================================================================
-- Excititor Module (uses 'vex' schema)
-- ============================================================================
GRANT USAGE ON SCHEMA vex TO excititor_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA vex TO excititor_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA vex TO excititor_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA vex GRANT ALL ON TABLES TO excititor_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA vex GRANT ALL ON SEQUENCES TO excititor_user;
-- ============================================================================
-- Scanner Module
-- ============================================================================
GRANT USAGE ON SCHEMA scanner TO scanner_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA scanner TO scanner_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA scanner TO scanner_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA scanner GRANT ALL ON TABLES TO scanner_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA scanner GRANT ALL ON SEQUENCES TO scanner_user;
-- ============================================================================
-- Scheduler Module
-- ============================================================================
GRANT USAGE ON SCHEMA scheduler TO scheduler_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA scheduler TO scheduler_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA scheduler TO scheduler_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA scheduler GRANT ALL ON TABLES TO scheduler_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA scheduler GRANT ALL ON SEQUENCES TO scheduler_user;
-- ============================================================================
-- TaskRunner Module
-- ============================================================================
GRANT USAGE ON SCHEMA taskrunner TO taskrunner_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA taskrunner TO taskrunner_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA taskrunner TO taskrunner_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA taskrunner GRANT ALL ON TABLES TO taskrunner_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA taskrunner GRANT ALL ON SEQUENCES TO taskrunner_user;
-- ============================================================================
-- Policy Module
-- ============================================================================
GRANT USAGE ON SCHEMA policy TO policy_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA policy TO policy_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA policy TO policy_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA policy GRANT ALL ON TABLES TO policy_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA policy GRANT ALL ON SEQUENCES TO policy_user;
-- ============================================================================
-- Unknowns Module
-- ============================================================================
GRANT USAGE ON SCHEMA unknowns TO unknowns_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA unknowns TO unknowns_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA unknowns TO unknowns_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA unknowns GRANT ALL ON TABLES TO unknowns_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA unknowns GRANT ALL ON SEQUENCES TO unknowns_user;
-- ============================================================================
-- Attestor Module (uses 'proofchain' and 'attestor' schemas)
-- ============================================================================
GRANT USAGE ON SCHEMA proofchain TO attestor_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA proofchain TO attestor_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA proofchain TO attestor_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA proofchain GRANT ALL ON TABLES TO attestor_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA proofchain GRANT ALL ON SEQUENCES TO attestor_user;
GRANT USAGE ON SCHEMA attestor TO attestor_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA attestor TO attestor_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA attestor TO attestor_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA attestor GRANT ALL ON TABLES TO attestor_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA attestor GRANT ALL ON SEQUENCES TO attestor_user;
-- ============================================================================
-- Signer Module
-- ============================================================================
GRANT USAGE ON SCHEMA signer TO signer_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA signer TO signer_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA signer TO signer_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA signer GRANT ALL ON TABLES TO signer_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA signer GRANT ALL ON SEQUENCES TO signer_user;
-- ============================================================================
-- Notify Module
-- ============================================================================
GRANT USAGE ON SCHEMA notify TO notify_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA notify TO notify_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA notify TO notify_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA notify GRANT ALL ON TABLES TO notify_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA notify GRANT ALL ON SEQUENCES TO notify_user;
-- ============================================================================
-- Signals Module
-- ============================================================================
GRANT USAGE ON SCHEMA signals TO signals_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA signals TO signals_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA signals TO signals_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA signals GRANT ALL ON TABLES TO signals_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA signals GRANT ALL ON SEQUENCES TO signals_user;
-- ============================================================================
-- Packs Registry Module
-- ============================================================================
GRANT USAGE ON SCHEMA packs TO packs_user;
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA packs TO packs_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA packs TO packs_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA packs GRANT ALL ON TABLES TO packs_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA packs GRANT ALL ON SEQUENCES TO packs_user;
-- ============================================================================
-- Verification
-- ============================================================================
DO $$
DECLARE
v_user TEXT;
v_schema TEXT;
BEGIN
RAISE NOTICE 'Per-module permissions granted:';
RAISE NOTICE ' authority_user -> authority';
RAISE NOTICE ' concelier_user -> vuln';
RAISE NOTICE ' excititor_user -> vex';
RAISE NOTICE ' scanner_user -> scanner';
RAISE NOTICE ' scheduler_user -> scheduler';
RAISE NOTICE ' taskrunner_user -> taskrunner';
RAISE NOTICE ' policy_user -> policy';
RAISE NOTICE ' unknowns_user -> unknowns';
RAISE NOTICE ' attestor_user -> proofchain, attestor';
RAISE NOTICE ' signer_user -> signer';
RAISE NOTICE ' notify_user -> notify';
RAISE NOTICE ' signals_user -> signals';
RAISE NOTICE ' packs_user -> packs';
END $$;