stela ops usage fixes roles propagation and timoeut, one account to support multi tenants, migrations consolidation, search to support documentation, doctor and open api vector db search

2026-02-22 19:27:54 +02:00
parent a29f438f53
commit bd8fee6ed8
373 changed files with 832097 additions and 3369 deletions
--- a/devops/compose/postgres-init/06-attestor-proofchain-schema.sql
+++ b/devops/compose/postgres-init/06-attestor-proofchain-schema.sql
@@ -0,0 +1,245 @@
+-- Attestor Schema Migration 001: Initial Schema (Compacted)
+-- Consolidated from 20251214000001_AddProofChainSchema.sql and 20251216_001_create_rekor_submission_queue.sql
+-- for 1.0.0 release
+-- Creates the proofchain schema for proof chain persistence and attestor schema for Rekor queue
+
+-- ============================================================================
+-- Extensions
+-- ============================================================================
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+-- ============================================================================
+-- Schema Creation
+-- ============================================================================
+
+CREATE SCHEMA IF NOT EXISTS proofchain;
+CREATE SCHEMA IF NOT EXISTS attestor;
+
+-- ============================================================================
+-- Enum Types
+-- ============================================================================
+
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'verification_result' AND typnamespace = 'proofchain'::regnamespace) THEN
+        CREATE TYPE proofchain.verification_result AS ENUM ('pass', 'fail', 'pending');
+    END IF;
+END $$;
+
+-- ============================================================================
+-- ProofChain Schema Tables
+-- ============================================================================
+
+-- Trust anchors table (create first - no dependencies)
+CREATE TABLE IF NOT EXISTS proofchain.trust_anchors (
+    anchor_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    purl_pattern TEXT NOT NULL,
+    allowed_keyids TEXT[] NOT NULL,
+    allowed_predicate_types TEXT[],
+    policy_ref TEXT,
+    policy_version TEXT,
+    revoked_keys TEXT[] DEFAULT '{}',
+    is_active BOOLEAN NOT NULL DEFAULT TRUE,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_trust_anchors_pattern ON proofchain.trust_anchors(purl_pattern);
+CREATE INDEX IF NOT EXISTS idx_trust_anchors_active ON proofchain.trust_anchors(is_active) WHERE is_active = TRUE;
+
+COMMENT ON TABLE proofchain.trust_anchors IS 'Trust anchor configurations for dependency verification';
+COMMENT ON COLUMN proofchain.trust_anchors.purl_pattern IS 'PURL glob pattern (e.g., pkg:npm/*)';
+COMMENT ON COLUMN proofchain.trust_anchors.revoked_keys IS 'Key IDs that have been revoked but may appear in old proofs';
+
+-- SBOM entries table
+CREATE TABLE IF NOT EXISTS proofchain.sbom_entries (
+    entry_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    bom_digest VARCHAR(64) NOT NULL,
+    purl TEXT NOT NULL,
+    version TEXT,
+    artifact_digest VARCHAR(64),
+    trust_anchor_id UUID REFERENCES proofchain.trust_anchors(anchor_id) ON DELETE SET NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    CONSTRAINT uq_sbom_entry UNIQUE (bom_digest, purl, version)
+);
+
+CREATE INDEX IF NOT EXISTS idx_sbom_entries_bom_digest ON proofchain.sbom_entries(bom_digest);
+CREATE INDEX IF NOT EXISTS idx_sbom_entries_purl ON proofchain.sbom_entries(purl);
+CREATE INDEX IF NOT EXISTS idx_sbom_entries_artifact ON proofchain.sbom_entries(artifact_digest);
+CREATE INDEX IF NOT EXISTS idx_sbom_entries_anchor ON proofchain.sbom_entries(trust_anchor_id);
+
+COMMENT ON TABLE proofchain.sbom_entries IS 'SBOM component entries with content-addressed identifiers';
+COMMENT ON COLUMN proofchain.sbom_entries.bom_digest IS 'SHA-256 hash of the parent SBOM document';
+COMMENT ON COLUMN proofchain.sbom_entries.purl IS 'Package URL (PURL) of the component';
+COMMENT ON COLUMN proofchain.sbom_entries.artifact_digest IS 'SHA-256 hash of the component artifact if available';
+
+-- DSSE envelopes table
+CREATE TABLE IF NOT EXISTS proofchain.dsse_envelopes (
+    env_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    entry_id UUID NOT NULL REFERENCES proofchain.sbom_entries(entry_id) ON DELETE CASCADE,
+    predicate_type TEXT NOT NULL,
+    signer_keyid TEXT NOT NULL,
+    body_hash VARCHAR(64) NOT NULL,
+    envelope_blob_ref TEXT NOT NULL,
+    signed_at TIMESTAMPTZ NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    CONSTRAINT uq_dsse_envelope UNIQUE (entry_id, predicate_type, body_hash)
+);
+
+CREATE INDEX IF NOT EXISTS idx_dsse_entry_predicate ON proofchain.dsse_envelopes(entry_id, predicate_type);
+CREATE INDEX IF NOT EXISTS idx_dsse_signer ON proofchain.dsse_envelopes(signer_keyid);
+CREATE INDEX IF NOT EXISTS idx_dsse_body_hash ON proofchain.dsse_envelopes(body_hash);
+
+COMMENT ON TABLE proofchain.dsse_envelopes IS 'Signed DSSE envelopes for proof chain statements';
+COMMENT ON COLUMN proofchain.dsse_envelopes.predicate_type IS 'Predicate type URI (e.g., evidence.stella/v1)';
+COMMENT ON COLUMN proofchain.dsse_envelopes.envelope_blob_ref IS 'Reference to blob storage (OCI, S3, file)';
+
+-- Spines table
+CREATE TABLE IF NOT EXISTS proofchain.spines (
+    entry_id UUID PRIMARY KEY REFERENCES proofchain.sbom_entries(entry_id) ON DELETE CASCADE,
+    bundle_id VARCHAR(64) NOT NULL,
+    evidence_ids TEXT[] NOT NULL,
+    reasoning_id VARCHAR(64) NOT NULL,
+    vex_id VARCHAR(64) NOT NULL,
+    anchor_id UUID REFERENCES proofchain.trust_anchors(anchor_id) ON DELETE SET NULL,
+    policy_version TEXT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    CONSTRAINT uq_spine_bundle UNIQUE (bundle_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_spines_bundle ON proofchain.spines(bundle_id);
+CREATE INDEX IF NOT EXISTS idx_spines_anchor ON proofchain.spines(anchor_id);
+CREATE INDEX IF NOT EXISTS idx_spines_policy ON proofchain.spines(policy_version);
+
+COMMENT ON TABLE proofchain.spines IS 'Proof spines linking evidence to verdicts via merkle aggregation';
+COMMENT ON COLUMN proofchain.spines.bundle_id IS 'ProofBundleID (merkle root of all components)';
+COMMENT ON COLUMN proofchain.spines.evidence_ids IS 'Array of EvidenceIDs in sorted order';
+
+-- Rekor entries table
+CREATE TABLE IF NOT EXISTS proofchain.rekor_entries (
+    dsse_sha256 VARCHAR(64) PRIMARY KEY,
+    log_index BIGINT NOT NULL,
+    log_id TEXT NOT NULL,
+    uuid TEXT NOT NULL,
+    integrated_time BIGINT NOT NULL,
+    inclusion_proof JSONB NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    env_id UUID REFERENCES proofchain.dsse_envelopes(env_id) ON DELETE SET NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_rekor_log_index ON proofchain.rekor_entries(log_index);
+CREATE INDEX IF NOT EXISTS idx_rekor_log_id ON proofchain.rekor_entries(log_id);
+CREATE INDEX IF NOT EXISTS idx_rekor_uuid ON proofchain.rekor_entries(uuid);
+CREATE INDEX IF NOT EXISTS idx_rekor_env ON proofchain.rekor_entries(env_id);
+
+COMMENT ON TABLE proofchain.rekor_entries IS 'Rekor transparency log entries for verification';
+COMMENT ON COLUMN proofchain.rekor_entries.inclusion_proof IS 'Merkle inclusion proof from Rekor';
+
+-- Audit log table
+CREATE TABLE IF NOT EXISTS proofchain.audit_log (
+    log_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    operation TEXT NOT NULL,
+    entity_type TEXT NOT NULL,
+    entity_id TEXT NOT NULL,
+    actor TEXT,
+    details JSONB,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_audit_entity ON proofchain.audit_log(entity_type, entity_id);
+CREATE INDEX IF NOT EXISTS idx_audit_created ON proofchain.audit_log(created_at DESC);
+
+COMMENT ON TABLE proofchain.audit_log IS 'Audit log for proof chain operations';
+
+-- ============================================================================
+-- Attestor Schema Tables
+-- ============================================================================
+
+-- Rekor submission queue table
+CREATE TABLE IF NOT EXISTS attestor.rekor_submission_queue (
+    id UUID PRIMARY KEY,
+    tenant_id TEXT NOT NULL,
+    bundle_sha256 TEXT NOT NULL,
+    dsse_payload BYTEA NOT NULL,
+    backend TEXT NOT NULL DEFAULT 'primary',
+
+    -- Status lifecycle: pending -> submitting -> submitted | retrying -> dead_letter
+    status TEXT NOT NULL DEFAULT 'pending'
+        CHECK (status IN ('pending', 'submitting', 'retrying', 'submitted', 'dead_letter')),
+
+    attempt_count INTEGER NOT NULL DEFAULT 0,
+    max_attempts INTEGER NOT NULL DEFAULT 5,
+    next_retry_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+    -- Populated on success
+    rekor_uuid TEXT,
+    rekor_index BIGINT,
+
+    -- Populated on failure
+    last_error TEXT
+);
+
+COMMENT ON TABLE attestor.rekor_submission_queue IS
+    'Durable retry queue for Rekor transparency log submissions';
+COMMENT ON COLUMN attestor.rekor_submission_queue.status IS
+    'Submission lifecycle: pending -> submitting -> (submitted | retrying -> dead_letter)';
+COMMENT ON COLUMN attestor.rekor_submission_queue.backend IS
+    'Target Rekor backend (primary or mirror)';
+COMMENT ON COLUMN attestor.rekor_submission_queue.dsse_payload IS
+    'Serialized DSSE envelope to submit';
+
+-- Index for dequeue operations (status + next_retry_at for SKIP LOCKED queries)
+CREATE INDEX IF NOT EXISTS idx_rekor_queue_dequeue
+    ON attestor.rekor_submission_queue (status, next_retry_at)
+    WHERE status IN ('pending', 'retrying');
+
+-- Index for tenant-scoped queries
+CREATE INDEX IF NOT EXISTS idx_rekor_queue_tenant
+    ON attestor.rekor_submission_queue (tenant_id);
+
+-- Index for bundle lookup (deduplication check)
+CREATE INDEX IF NOT EXISTS idx_rekor_queue_bundle
+    ON attestor.rekor_submission_queue (tenant_id, bundle_sha256);
+
+-- Index for dead letter management
+CREATE INDEX IF NOT EXISTS idx_rekor_queue_dead_letter
+    ON attestor.rekor_submission_queue (status, updated_at)
+    WHERE status = 'dead_letter';
+
+-- Index for cleanup of completed submissions
+CREATE INDEX IF NOT EXISTS idx_rekor_queue_completed
+    ON attestor.rekor_submission_queue (status, updated_at)
+    WHERE status = 'submitted';
+
+-- ============================================================================
+-- Trigger Functions
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION proofchain.update_updated_at_column()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = NOW();
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Apply updated_at trigger to trust_anchors
+DROP TRIGGER IF EXISTS update_trust_anchors_updated_at ON proofchain.trust_anchors;
+CREATE TRIGGER update_trust_anchors_updated_at
+    BEFORE UPDATE ON proofchain.trust_anchors
+    FOR EACH ROW
+    EXECUTE FUNCTION proofchain.update_updated_at_column();
+
+-- Apply updated_at trigger to rekor_submission_queue
+DROP TRIGGER IF EXISTS update_rekor_queue_updated_at ON attestor.rekor_submission_queue;
+CREATE TRIGGER update_rekor_queue_updated_at
+    BEFORE UPDATE ON attestor.rekor_submission_queue
+    FOR EACH ROW
+    EXECUTE FUNCTION proofchain.update_updated_at_column();