stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

devops folders consolidate

Browse Source
This commit is contained in:
master
2026-01-25 23:27:41 +02:00
parent 6e687b523a
commit a50bbb38ef
334 changed files with 35079 additions and 5569 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
deploy/database/migrations/005_timestamp_evidence.sql Normal file
View File

@@ -0,0 +1,69 @@
-- -----------------------------------------------------------------------------
-- 005_timestamp_evidence.sql
-- Sprint: SPRINT_20260119_009 Evidence Storage for Timestamps
-- Task: EVT-002 - PostgreSQL Schema Extension
-- Description: Schema for storing timestamp and revocation evidence.
-- -----------------------------------------------------------------------------
-- Ensure the evidence schema exists
CREATE SCHEMA IF NOT EXISTS evidence;
-- Timestamp evidence storage
CREATE TABLE IF NOT EXISTS evidence.timestamp_tokens (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
artifact_digest TEXT NOT NULL,
digest_algorithm TEXT NOT NULL,
tst_blob BYTEA NOT NULL,
generation_time TIMESTAMPTZ NOT NULL,
tsa_name TEXT NOT NULL,
tsa_policy_oid TEXT NOT NULL,
serial_number TEXT NOT NULL,
tsa_chain_pem TEXT NOT NULL,
ocsp_response BYTEA,
crl_snapshot BYTEA,
captured_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
provider_name TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT uq_timestamp_artifact_time UNIQUE (artifact_digest, generation_time)
);
-- Indexes for timestamp queries
CREATE INDEX IF NOT EXISTS idx_timestamp_artifact ON evidence.timestamp_tokens(artifact_digest);
CREATE INDEX IF NOT EXISTS idx_timestamp_generation ON evidence.timestamp_tokens(generation_time);
CREATE INDEX IF NOT EXISTS idx_timestamp_provider ON evidence.timestamp_tokens(provider_name);
CREATE INDEX IF NOT EXISTS idx_timestamp_created ON evidence.timestamp_tokens(created_at);
-- Revocation evidence storage
CREATE TABLE IF NOT EXISTS evidence.revocation_snapshots (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
certificate_fingerprint TEXT NOT NULL,
source TEXT NOT NULL CHECK (source IN ('Ocsp', 'Crl', 'None')),
raw_response BYTEA NOT NULL,
response_time TIMESTAMPTZ NOT NULL,
valid_until TIMESTAMPTZ NOT NULL,
status TEXT NOT NULL CHECK (status IN ('Good', 'Revoked', 'Unknown')),
revocation_time TIMESTAMPTZ,
reason TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- Indexes for revocation queries
CREATE INDEX IF NOT EXISTS idx_revocation_cert ON evidence.revocation_snapshots(certificate_fingerprint);
CREATE INDEX IF NOT EXISTS idx_revocation_valid ON evidence.revocation_snapshots(valid_until);
CREATE INDEX IF NOT EXISTS idx_revocation_status ON evidence.revocation_snapshots(status);
CREATE INDEX IF NOT EXISTS idx_revocation_created ON evidence.revocation_snapshots(created_at);
-- Comments
COMMENT ON TABLE evidence.timestamp_tokens IS 'RFC-3161 TimeStampToken evidence for long-term validation';
COMMENT ON TABLE evidence.revocation_snapshots IS 'OCSP/CRL certificate revocation evidence snapshots';
COMMENT ON COLUMN evidence.timestamp_tokens.artifact_digest IS 'SHA-256 digest of the timestamped artifact';
COMMENT ON COLUMN evidence.timestamp_tokens.tst_blob IS 'Raw DER-encoded RFC 3161 TimeStampToken';
COMMENT ON COLUMN evidence.timestamp_tokens.tsa_chain_pem IS 'PEM-encoded TSA certificate chain for LTV';
COMMENT ON COLUMN evidence.timestamp_tokens.ocsp_response IS 'Stapled OCSP response at signing time';
COMMENT ON COLUMN evidence.timestamp_tokens.crl_snapshot IS 'CRL snapshot at signing time (fallback for OCSP)';
COMMENT ON COLUMN evidence.revocation_snapshots.certificate_fingerprint IS 'SHA-256 fingerprint of the certificate';
COMMENT ON COLUMN evidence.revocation_snapshots.raw_response IS 'Raw OCSP response or CRL bytes';
COMMENT ON COLUMN evidence.revocation_snapshots.response_time IS 'thisUpdate from the response';
COMMENT ON COLUMN evidence.revocation_snapshots.valid_until IS 'nextUpdate from the response';

21
deploy/database/migrations/005_timestamp_evidence_rollback.sql Normal file
View File

@@ -0,0 +1,21 @@
-- -----------------------------------------------------------------------------
-- 005_timestamp_evidence_rollback.sql
-- Sprint: SPRINT_20260119_009 Evidence Storage for Timestamps
-- Task: EVT-002 - PostgreSQL Schema Extension
-- Description: Rollback migration for timestamp and revocation evidence.
-- -----------------------------------------------------------------------------
-- Drop indexes first
DROP INDEX IF EXISTS evidence.idx_timestamp_artifact;
DROP INDEX IF EXISTS evidence.idx_timestamp_generation;
DROP INDEX IF EXISTS evidence.idx_timestamp_provider;
DROP INDEX IF EXISTS evidence.idx_timestamp_created;
DROP INDEX IF EXISTS evidence.idx_revocation_cert;
DROP INDEX IF EXISTS evidence.idx_revocation_valid;
DROP INDEX IF EXISTS evidence.idx_revocation_status;
DROP INDEX IF EXISTS evidence.idx_revocation_created;
-- Drop tables
DROP TABLE IF EXISTS evidence.revocation_snapshots;
DROP TABLE IF EXISTS evidence.timestamp_tokens;

120
deploy/database/migrations/005_validation_harness.sql Normal file
View File

@@ -0,0 +1,120 @@
-- Validation harness schema for tracking validation runs and match results
-- Migration: 005_validation_harness.sql
-- Validation runs table
CREATE TABLE IF NOT EXISTS groundtruth.validation_runs (
run_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
description TEXT,
status TEXT NOT NULL DEFAULT 'pending',
-- Configuration (stored as JSONB)
config JSONB NOT NULL,
-- Timestamps
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
-- Metrics (populated after completion)
total_pairs INT,
total_functions INT,
true_positives INT,
false_positives INT,
true_negatives INT,
false_negatives INT,
match_rate DOUBLE PRECISION,
precision_score DOUBLE PRECISION,
recall_score DOUBLE PRECISION,
f1_score DOUBLE PRECISION,
average_match_score DOUBLE PRECISION,
-- Mismatch counts by bucket (JSONB map)
mismatch_counts JSONB,
-- Metadata
corpus_snapshot_id TEXT,
matcher_version TEXT,
error_message TEXT,
tags TEXT[] DEFAULT '{}',
-- Constraints
CONSTRAINT valid_status CHECK (status IN ('pending', 'running', 'completed', 'failed', 'cancelled'))
);
-- Indexes for validation runs
CREATE INDEX IF NOT EXISTS idx_validation_runs_status ON groundtruth.validation_runs(status);
CREATE INDEX IF NOT EXISTS idx_validation_runs_created_at ON groundtruth.validation_runs(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_validation_runs_tags ON groundtruth.validation_runs USING GIN (tags);
-- Match results table
CREATE TABLE IF NOT EXISTS groundtruth.match_results (
result_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
run_id UUID NOT NULL REFERENCES groundtruth.validation_runs(run_id) ON DELETE CASCADE,
security_pair_id UUID NOT NULL,
-- Source function
source_name TEXT NOT NULL,
source_demangled_name TEXT,
source_address BIGINT NOT NULL,
source_size BIGINT,
source_build_id TEXT NOT NULL,
source_binary_name TEXT NOT NULL,
-- Expected target
expected_name TEXT NOT NULL,
expected_demangled_name TEXT,
expected_address BIGINT NOT NULL,
expected_size BIGINT,
expected_build_id TEXT NOT NULL,
expected_binary_name TEXT NOT NULL,
-- Actual matched target (nullable if no match found)
actual_name TEXT,
actual_demangled_name TEXT,
actual_address BIGINT,
actual_size BIGINT,
actual_build_id TEXT,
actual_binary_name TEXT,
-- Outcome
outcome TEXT NOT NULL,
match_score DOUBLE PRECISION,
confidence TEXT,
-- Mismatch analysis
inferred_cause TEXT,
mismatch_detail JSONB,
-- Performance
match_duration_ms DOUBLE PRECISION,
-- Constraints
CONSTRAINT valid_outcome CHECK (outcome IN ('true_positive', 'false_positive', 'true_negative', 'false_negative'))
);
-- Indexes for match results
CREATE INDEX IF NOT EXISTS idx_match_results_run_id ON groundtruth.match_results(run_id);
CREATE INDEX IF NOT EXISTS idx_match_results_security_pair_id ON groundtruth.match_results(security_pair_id);
CREATE INDEX IF NOT EXISTS idx_match_results_outcome ON groundtruth.match_results(outcome);
CREATE INDEX IF NOT EXISTS idx_match_results_inferred_cause ON groundtruth.match_results(inferred_cause) WHERE inferred_cause IS NOT NULL;
-- View for run summaries
CREATE OR REPLACE VIEW groundtruth.validation_run_summaries AS
SELECT
run_id AS id,
name,
status,
created_at,
completed_at,
match_rate,
f1_score,
total_pairs AS pair_count,
total_functions AS function_count,
tags
FROM groundtruth.validation_runs;
-- Comments
COMMENT ON TABLE groundtruth.validation_runs IS 'Validation harness runs with aggregate metrics';
COMMENT ON TABLE groundtruth.match_results IS 'Per-function match results from validation runs';
COMMENT ON VIEW groundtruth.validation_run_summaries IS 'Summary view for listing validation runs';

27
deploy/database/migrations/006_timestamp_supersession.sql Normal file
View File

@@ -0,0 +1,27 @@
-- -----------------------------------------------------------------------------
-- 006_timestamp_supersession.sql
-- Sprint: SPRINT_20260119_009 Evidence Storage for Timestamps
-- Task: EVT-005 - Re-Timestamping Support
-- Description: Schema extension for timestamp supersession chain.
-- -----------------------------------------------------------------------------
-- Add supersession column for re-timestamping chain
ALTER TABLE evidence.timestamp_tokens
ADD COLUMN IF NOT EXISTS supersedes_id UUID REFERENCES evidence.timestamp_tokens(id);
-- Index for finding superseding timestamps
CREATE INDEX IF NOT EXISTS idx_timestamp_supersedes ON evidence.timestamp_tokens(supersedes_id);
-- Index for finding timestamps by expiry (for re-timestamp scheduling)
-- Note: We need to track TSA certificate expiry separately - for now use generation_time + typical cert lifetime
CREATE INDEX IF NOT EXISTS idx_timestamp_for_retimestamp
ON evidence.timestamp_tokens(generation_time)
WHERE supersedes_id IS NULL; -- Only query leaf timestamps (not already superseded)
-- Comments
COMMENT ON COLUMN evidence.timestamp_tokens.supersedes_id IS 'ID of the timestamp this supersedes (for re-timestamping chain)';
-- Rollback script (execute separately if needed):
-- ALTER TABLE evidence.timestamp_tokens DROP COLUMN IF EXISTS supersedes_id;
-- DROP INDEX IF EXISTS evidence.idx_timestamp_supersedes;
-- DROP INDEX IF EXISTS evidence.idx_timestamp_for_retimestamp;

108
deploy/database/migrations/V20260108__opsmemory_advisoryai_schema.sql Normal file
View File

@@ -0,0 +1,108 @@
-- OpsMemory and AdvisoryAI PostgreSQL Schema Migration
-- Version: 20260108
-- Author: StellaOps Agent
-- Sprint: SPRINT_20260107_006_004 (OpsMemory), SPRINT_20260107_006_003 (AdvisoryAI)
-- ============================================================================
-- OpsMemory Schema
-- ============================================================================
CREATE SCHEMA IF NOT EXISTS opsmemory;
-- Decision records table
CREATE TABLE IF NOT EXISTS opsmemory.decisions (
memory_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
recorded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-- Situation context
cve_id TEXT,
component_purl TEXT,
severity TEXT,
reachability TEXT,
epss_score DECIMAL(5, 4),
cvss_score DECIMAL(3, 1),
context_tags TEXT[],
similarity_vector DOUBLE PRECISION[],
-- Decision details
action TEXT NOT NULL,
rationale TEXT,
decided_by TEXT NOT NULL,
policy_reference TEXT,
mitigation_type TEXT,
mitigation_details TEXT,
-- Outcome (nullable until recorded)
outcome_status TEXT,
resolution_time INTERVAL,
actual_impact TEXT,
lessons_learned TEXT,
outcome_recorded_by TEXT,
outcome_recorded_at TIMESTAMPTZ
);
-- Indexes for querying
CREATE INDEX IF NOT EXISTS idx_opsmemory_decisions_tenant ON opsmemory.decisions(tenant_id);
CREATE INDEX IF NOT EXISTS idx_opsmemory_decisions_cve ON opsmemory.decisions(cve_id);
CREATE INDEX IF NOT EXISTS idx_opsmemory_decisions_component ON opsmemory.decisions(component_purl);
CREATE INDEX IF NOT EXISTS idx_opsmemory_decisions_recorded ON opsmemory.decisions(recorded_at);
CREATE INDEX IF NOT EXISTS idx_opsmemory_decisions_action ON opsmemory.decisions(action);
CREATE INDEX IF NOT EXISTS idx_opsmemory_decisions_outcome ON opsmemory.decisions(outcome_status);
-- ============================================================================
-- AdvisoryAI Schema
-- ============================================================================
CREATE SCHEMA IF NOT EXISTS advisoryai;
-- Conversations table
CREATE TABLE IF NOT EXISTS advisoryai.conversations (
conversation_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
user_id TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
context JSONB,
metadata JSONB
);
-- Conversation turns table
CREATE TABLE IF NOT EXISTS advisoryai.turns (
turn_id TEXT PRIMARY KEY,
conversation_id TEXT NOT NULL REFERENCES advisoryai.conversations(conversation_id) ON DELETE CASCADE,
role TEXT NOT NULL,
content TEXT NOT NULL,
timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(),
evidence_links JSONB,
proposed_actions JSONB,
metadata JSONB
);
-- Indexes for querying
CREATE INDEX IF NOT EXISTS idx_advisoryai_conv_tenant ON advisoryai.conversations(tenant_id);
CREATE INDEX IF NOT EXISTS idx_advisoryai_conv_user ON advisoryai.conversations(user_id);
CREATE INDEX IF NOT EXISTS idx_advisoryai_conv_updated ON advisoryai.conversations(updated_at);
CREATE INDEX IF NOT EXISTS idx_advisoryai_turns_conv ON advisoryai.turns(conversation_id);
CREATE INDEX IF NOT EXISTS idx_advisoryai_turns_timestamp ON advisoryai.turns(timestamp);
-- ============================================================================
-- Comments for documentation
-- ============================================================================
COMMENT ON SCHEMA opsmemory IS 'OpsMemory: Decision ledger for security playbook learning';
COMMENT ON SCHEMA advisoryai IS 'AdvisoryAI: Chat conversation storage';
COMMENT ON TABLE opsmemory.decisions IS 'Stores security decisions and their outcomes for playbook suggestions';
COMMENT ON TABLE advisoryai.conversations IS 'Stores AI chat conversations with context';
COMMENT ON TABLE advisoryai.turns IS 'Individual messages in conversations';
-- ============================================================================
-- Grants (adjust as needed for your environment)
-- ============================================================================
-- GRANT USAGE ON SCHEMA opsmemory TO stellaops_app;
-- GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA opsmemory TO stellaops_app;
-- GRANT USAGE ON SCHEMA advisoryai TO stellaops_app;
-- GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA advisoryai TO stellaops_app;

220
deploy/database/migrations/V20260110__reachability_cve_mapping_schema.sql Normal file
View File

@@ -0,0 +1,220 @@
-- CVE-Symbol Mapping PostgreSQL Schema Migration
-- Version: 20260110
-- Author: StellaOps Agent
-- Sprint: SPRINT_20260109_009_003_BE_cve_symbol_mapping
-- ============================================================================
-- Reachability Schema
-- ============================================================================
CREATE SCHEMA IF NOT EXISTS reachability;
-- ============================================================================
-- CVE-Symbol Mapping Tables
-- ============================================================================
-- Mapping source enumeration type
CREATE TYPE reachability.mapping_source AS ENUM (
'patch_analysis',
'osv_advisory',
'nvd_cpe',
'manual_curation',
'fuzzing_corpus',
'exploit_database',
'unknown'
);
-- Vulnerability type enumeration (for taint analysis)
CREATE TYPE reachability.vulnerability_type AS ENUM (
'source',
'sink',
'gadget',
'both_source_and_sink',
'unknown'
);
-- Main CVE-symbol mapping table
CREATE TABLE IF NOT EXISTS reachability.cve_symbol_mappings (
mapping_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-- CVE identification
cve_id TEXT NOT NULL,
cve_id_normalized TEXT NOT NULL GENERATED ALWAYS AS (UPPER(cve_id)) STORED,
-- Affected package (PURL format)
purl TEXT NOT NULL,
affected_versions TEXT[], -- Version ranges like [">=1.0.0,<2.0.0"]
fixed_versions TEXT[], -- Versions where fix is applied
-- Vulnerable symbol details
symbol_name TEXT NOT NULL,
canonical_id TEXT, -- Normalized symbol ID from canonicalization service
file_path TEXT,
start_line INTEGER,
end_line INTEGER,
-- Metadata
source reachability.mapping_source NOT NULL DEFAULT 'unknown',
vulnerability_type reachability.vulnerability_type NOT NULL DEFAULT 'unknown',
confidence DECIMAL(3, 2) NOT NULL DEFAULT 0.5 CHECK (confidence >= 0 AND confidence <= 1),
-- Provenance
evidence_uri TEXT, -- stella:// URI to evidence
source_commit_url TEXT,
patch_url TEXT,
-- Timestamps
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
verified_at TIMESTAMPTZ,
verified_by TEXT,
-- Tenant support
tenant_id TEXT NOT NULL DEFAULT 'default'
);
-- Vulnerable symbol detail records (for additional symbol metadata)
CREATE TABLE IF NOT EXISTS reachability.vulnerable_symbols (
symbol_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
mapping_id UUID NOT NULL REFERENCES reachability.cve_symbol_mappings(mapping_id) ON DELETE CASCADE,
-- Symbol identification
symbol_name TEXT NOT NULL,
canonical_id TEXT,
symbol_type TEXT, -- 'function', 'method', 'class', 'module'
-- Location
file_path TEXT,
start_line INTEGER,
end_line INTEGER,
-- Code context
signature TEXT, -- Function signature
containing_class TEXT,
namespace TEXT,
-- Vulnerability context
vulnerability_type reachability.vulnerability_type NOT NULL DEFAULT 'unknown',
is_entry_point BOOLEAN DEFAULT FALSE,
requires_control_flow BOOLEAN DEFAULT FALSE,
-- Metadata
confidence DECIMAL(3, 2) NOT NULL DEFAULT 0.5,
notes TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- Patch analysis results (cached)
CREATE TABLE IF NOT EXISTS reachability.patch_analysis (
analysis_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-- Source identification
commit_url TEXT NOT NULL UNIQUE,
repository_url TEXT,
commit_sha TEXT,
-- Analysis results (stored as JSONB for flexibility)
diff_content TEXT,
extracted_symbols JSONB NOT NULL DEFAULT '[]',
language_detected TEXT,
-- Metadata
analyzed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
analyzer_version TEXT,
-- Error tracking
analysis_status TEXT NOT NULL DEFAULT 'pending',
error_message TEXT
);
-- ============================================================================
-- Indexes
-- ============================================================================
-- CVE lookup indexes
CREATE INDEX IF NOT EXISTS idx_cve_mapping_cve_normalized ON reachability.cve_symbol_mappings(cve_id_normalized);
CREATE INDEX IF NOT EXISTS idx_cve_mapping_purl ON reachability.cve_symbol_mappings(purl);
CREATE INDEX IF NOT EXISTS idx_cve_mapping_symbol ON reachability.cve_symbol_mappings(symbol_name);
CREATE INDEX IF NOT EXISTS idx_cve_mapping_canonical ON reachability.cve_symbol_mappings(canonical_id) WHERE canonical_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_cve_mapping_tenant ON reachability.cve_symbol_mappings(tenant_id);
CREATE INDEX IF NOT EXISTS idx_cve_mapping_source ON reachability.cve_symbol_mappings(source);
CREATE INDEX IF NOT EXISTS idx_cve_mapping_confidence ON reachability.cve_symbol_mappings(confidence);
CREATE INDEX IF NOT EXISTS idx_cve_mapping_created ON reachability.cve_symbol_mappings(created_at);
-- Composite index for common queries
CREATE INDEX IF NOT EXISTS idx_cve_mapping_cve_purl ON reachability.cve_symbol_mappings(cve_id_normalized, purl);
-- Symbol indexes
CREATE INDEX IF NOT EXISTS idx_vuln_symbol_mapping ON reachability.vulnerable_symbols(mapping_id);
CREATE INDEX IF NOT EXISTS idx_vuln_symbol_name ON reachability.vulnerable_symbols(symbol_name);
CREATE INDEX IF NOT EXISTS idx_vuln_symbol_canonical ON reachability.vulnerable_symbols(canonical_id) WHERE canonical_id IS NOT NULL;
-- Patch analysis indexes
CREATE INDEX IF NOT EXISTS idx_patch_analysis_commit ON reachability.patch_analysis(commit_sha);
CREATE INDEX IF NOT EXISTS idx_patch_analysis_repo ON reachability.patch_analysis(repository_url);
-- ============================================================================
-- Full-text search
-- ============================================================================
-- Add tsvector column for symbol search
ALTER TABLE reachability.cve_symbol_mappings
ADD COLUMN IF NOT EXISTS symbol_search_vector tsvector
GENERATED ALWAYS AS (to_tsvector('simple', coalesce(symbol_name, '') || ' ' || coalesce(file_path, ''))) STORED;
CREATE INDEX IF NOT EXISTS idx_cve_mapping_fts ON reachability.cve_symbol_mappings USING GIN(symbol_search_vector);
-- ============================================================================
-- Trigger for updated_at
-- ============================================================================
CREATE OR REPLACE FUNCTION reachability.update_modified_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER update_cve_mapping_modtime
BEFORE UPDATE ON reachability.cve_symbol_mappings
FOR EACH ROW
EXECUTE FUNCTION reachability.update_modified_column();
-- ============================================================================
-- Comments for documentation
-- ============================================================================
COMMENT ON SCHEMA reachability IS 'Hybrid reachability analysis: CVE-symbol mappings, static/runtime evidence';
COMMENT ON TABLE reachability.cve_symbol_mappings IS 'Maps CVE IDs to vulnerable symbols with confidence scores';
COMMENT ON COLUMN reachability.cve_symbol_mappings.cve_id_normalized IS 'Uppercase normalized CVE ID for case-insensitive lookup';
COMMENT ON COLUMN reachability.cve_symbol_mappings.canonical_id IS 'Symbol canonical ID from canonicalization service';
COMMENT ON COLUMN reachability.cve_symbol_mappings.evidence_uri IS 'stella:// URI pointing to evidence bundle';
COMMENT ON TABLE reachability.vulnerable_symbols IS 'Additional symbol details for a CVE mapping';
COMMENT ON TABLE reachability.patch_analysis IS 'Cached patch analysis results for commit URLs';
-- ============================================================================
-- Initial data / seed (optional well-known CVEs for testing)
-- ============================================================================
-- Example: Log4Shell (CVE-2021-44228)
INSERT INTO reachability.cve_symbol_mappings (cve_id, purl, symbol_name, file_path, source, confidence, vulnerability_type)
VALUES
('CVE-2021-44228', 'pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1', 'JndiLookup.lookup', 'log4j-core/src/main/java/org/apache/logging/log4j/core/lookup/JndiLookup.java', 'manual_curation', 0.99, 'sink'),
('CVE-2021-44228', 'pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1', 'JndiManager.lookup', 'log4j-core/src/main/java/org/apache/logging/log4j/core/net/JndiManager.java', 'manual_curation', 0.95, 'sink')
ON CONFLICT DO NOTHING;
-- Example: Spring4Shell (CVE-2022-22965)
INSERT INTO reachability.cve_symbol_mappings (cve_id, purl, symbol_name, file_path, source, confidence, vulnerability_type)
VALUES
('CVE-2022-22965', 'pkg:maven/org.springframework/spring-beans@5.3.17', 'CachedIntrospectionResults.getBeanInfo', 'spring-beans/src/main/java/org/springframework/beans/CachedIntrospectionResults.java', 'patch_analysis', 0.90, 'source')
ON CONFLICT DO NOTHING;
-- Example: polyfill.io supply chain (CVE-2024-38526)
INSERT INTO reachability.cve_symbol_mappings (cve_id, purl, symbol_name, source, confidence, vulnerability_type)
VALUES
('CVE-2024-38526', 'pkg:npm/polyfill.io', 'window.polyfill', 'manual_curation', 0.85, 'source')
ON CONFLICT DO NOTHING;

38
deploy/database/migrations/V20260117__create_doctor_reports_table.sql Normal file
View File

@@ -0,0 +1,38 @@
-- -----------------------------------------------------------------------------
-- V20260117__create_doctor_reports_table.sql
-- Sprint: SPRINT_20260117_025_Doctor_coverage_expansion
-- Task: DOC-EXP-005 - Persistent Report Storage
-- Description: Migration to create doctor_reports table for persistent storage
-- -----------------------------------------------------------------------------
-- Doctor reports table for persistent storage
CREATE TABLE IF NOT EXISTS doctor_reports (
run_id VARCHAR(64) PRIMARY KEY,
started_at TIMESTAMPTZ NOT NULL,
completed_at TIMESTAMPTZ,
overall_severity VARCHAR(16) NOT NULL,
passed_count INTEGER NOT NULL DEFAULT 0,
warning_count INTEGER NOT NULL DEFAULT 0,
failed_count INTEGER NOT NULL DEFAULT 0,
skipped_count INTEGER NOT NULL DEFAULT 0,
info_count INTEGER NOT NULL DEFAULT 0,
total_count INTEGER NOT NULL DEFAULT 0,
report_json_compressed BYTEA NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- Index for listing reports by date
CREATE INDEX IF NOT EXISTS idx_doctor_reports_started_at
ON doctor_reports (started_at DESC);
-- Index for retention cleanup
CREATE INDEX IF NOT EXISTS idx_doctor_reports_created_at
ON doctor_reports (created_at);
-- Index for filtering by severity
CREATE INDEX IF NOT EXISTS idx_doctor_reports_severity
ON doctor_reports (overall_severity);
-- Comment on table
COMMENT ON TABLE doctor_reports IS 'Stores Doctor diagnostic reports with compression for audit trail';
COMMENT ON COLUMN doctor_reports.report_json_compressed IS 'GZip compressed JSON report data';

153
deploy/database/migrations/V20260117__vex_rekor_linkage.sql Normal file
View File

@@ -0,0 +1,153 @@
-- Migration: V20260117__vex_rekor_linkage.sql
-- Sprint: SPRINT_20260117_002_EXCITITOR_vex_rekor_linkage
-- Task: VRL-004, VRL-005 - Create Excititor and VexHub database migrations
-- Description: Add Rekor transparency log linkage columns to VEX tables
-- Author: StellaOps
-- Date: 2026-01-17
-- ============================================================================
-- EXCITITOR SCHEMA: vex_observations table
-- ============================================================================
-- Add Rekor linkage columns to vex_observations
ALTER TABLE IF EXISTS excititor.vex_observations
ADD COLUMN IF NOT EXISTS rekor_uuid TEXT,
ADD COLUMN IF NOT EXISTS rekor_log_index BIGINT,
ADD COLUMN IF NOT EXISTS rekor_integrated_time TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS rekor_log_url TEXT,
ADD COLUMN IF NOT EXISTS rekor_tree_root TEXT,
ADD COLUMN IF NOT EXISTS rekor_tree_size BIGINT,
ADD COLUMN IF NOT EXISTS rekor_inclusion_proof JSONB,
ADD COLUMN IF NOT EXISTS rekor_entry_body_hash TEXT,
ADD COLUMN IF NOT EXISTS rekor_entry_kind TEXT,
ADD COLUMN IF NOT EXISTS rekor_linked_at TIMESTAMPTZ;
-- Index for Rekor queries by UUID
CREATE INDEX IF NOT EXISTS idx_vex_observations_rekor_uuid
ON excititor.vex_observations(rekor_uuid)
WHERE rekor_uuid IS NOT NULL;
-- Index for Rekor queries by log index (for ordered traversal)
CREATE INDEX IF NOT EXISTS idx_vex_observations_rekor_log_index
ON excititor.vex_observations(rekor_log_index DESC)
WHERE rekor_log_index IS NOT NULL;
-- Index for finding unlinked observations (for retry/backfill)
CREATE INDEX IF NOT EXISTS idx_vex_observations_pending_rekor
ON excititor.vex_observations(created_at)
WHERE rekor_uuid IS NULL;
-- Comment on columns
COMMENT ON COLUMN excititor.vex_observations.rekor_uuid IS 'Rekor entry UUID (64-char hex)';
COMMENT ON COLUMN excititor.vex_observations.rekor_log_index IS 'Monotonically increasing log position';
COMMENT ON COLUMN excititor.vex_observations.rekor_integrated_time IS 'Time entry was integrated into Rekor log';
COMMENT ON COLUMN excititor.vex_observations.rekor_log_url IS 'Rekor server URL where entry was submitted';
COMMENT ON COLUMN excititor.vex_observations.rekor_tree_root IS 'Merkle tree root hash at submission time (base64)';
COMMENT ON COLUMN excititor.vex_observations.rekor_tree_size IS 'Tree size at submission time';
COMMENT ON COLUMN excititor.vex_observations.rekor_inclusion_proof IS 'RFC 6962 inclusion proof for offline verification';
COMMENT ON COLUMN excititor.vex_observations.rekor_entry_body_hash IS 'SHA-256 hash of entry body';
COMMENT ON COLUMN excititor.vex_observations.rekor_entry_kind IS 'Entry kind (dsse, intoto, hashedrekord)';
COMMENT ON COLUMN excititor.vex_observations.rekor_linked_at IS 'When linkage was recorded locally';
-- ============================================================================
-- EXCITITOR SCHEMA: vex_statement_change_events table
-- ============================================================================
-- Add Rekor linkage to change events
ALTER TABLE IF EXISTS excititor.vex_statement_change_events
ADD COLUMN IF NOT EXISTS rekor_entry_id TEXT,
ADD COLUMN IF NOT EXISTS rekor_log_index BIGINT;
-- Index for Rekor queries on change events
CREATE INDEX IF NOT EXISTS idx_vex_change_events_rekor
ON excititor.vex_statement_change_events(rekor_entry_id)
WHERE rekor_entry_id IS NOT NULL;
COMMENT ON COLUMN excititor.vex_statement_change_events.rekor_entry_id IS 'Rekor entry UUID for change attestation';
COMMENT ON COLUMN excititor.vex_statement_change_events.rekor_log_index IS 'Rekor log index for change attestation';
-- ============================================================================
-- VEXHUB SCHEMA: vex_statements table
-- ============================================================================
-- Add Rekor linkage columns to vex_statements
ALTER TABLE IF EXISTS vexhub.vex_statements
ADD COLUMN IF NOT EXISTS rekor_uuid TEXT,
ADD COLUMN IF NOT EXISTS rekor_log_index BIGINT,
ADD COLUMN IF NOT EXISTS rekor_integrated_time TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS rekor_inclusion_proof JSONB;
-- Index for Rekor queries
CREATE INDEX IF NOT EXISTS idx_vexhub_statements_rekor_uuid
ON vexhub.vex_statements(rekor_uuid)
WHERE rekor_uuid IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_vexhub_statements_rekor_log_index
ON vexhub.vex_statements(rekor_log_index DESC)
WHERE rekor_log_index IS NOT NULL;
COMMENT ON COLUMN vexhub.vex_statements.rekor_uuid IS 'Rekor entry UUID for statement attestation';
COMMENT ON COLUMN vexhub.vex_statements.rekor_log_index IS 'Rekor log index for statement attestation';
COMMENT ON COLUMN vexhub.vex_statements.rekor_integrated_time IS 'Time statement was integrated into Rekor log';
COMMENT ON COLUMN vexhub.vex_statements.rekor_inclusion_proof IS 'RFC 6962 inclusion proof for offline verification';
-- ============================================================================
-- ATTESTOR SCHEMA: rekor_entries verification tracking
-- Sprint: SPRINT_20260117_001_ATTESTOR_periodic_rekor_verification (PRV-003)
-- ============================================================================
-- Add verification tracking columns to existing rekor_entries table
ALTER TABLE IF EXISTS attestor.rekor_entries
ADD COLUMN IF NOT EXISTS last_verified_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS verification_count INT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS last_verification_result TEXT;
-- Index for verification queries (find entries needing verification)
CREATE INDEX IF NOT EXISTS idx_rekor_entries_verification
ON attestor.rekor_entries(created_at DESC, last_verified_at NULLS FIRST)
WHERE last_verification_result IS DISTINCT FROM 'invalid';
-- Index for finding never-verified entries
CREATE INDEX IF NOT EXISTS idx_rekor_entries_unverified
ON attestor.rekor_entries(created_at DESC)
WHERE last_verified_at IS NULL;
COMMENT ON COLUMN attestor.rekor_entries.last_verified_at IS 'Timestamp of last successful verification';
COMMENT ON COLUMN attestor.rekor_entries.verification_count IS 'Number of times entry has been verified';
COMMENT ON COLUMN attestor.rekor_entries.last_verification_result IS 'Result of last verification: valid, invalid, skipped';
-- ============================================================================
-- ATTESTOR SCHEMA: rekor_root_checkpoints table
-- Stores tree root checkpoints for consistency verification
-- ============================================================================
CREATE TABLE IF NOT EXISTS attestor.rekor_root_checkpoints (
id BIGSERIAL PRIMARY KEY,
tree_root TEXT NOT NULL,
tree_size BIGINT NOT NULL,
log_id TEXT NOT NULL,
log_url TEXT,
checkpoint_envelope TEXT,
captured_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
verified_at TIMESTAMPTZ,
is_consistent BOOLEAN,
inconsistency_reason TEXT,
CONSTRAINT uq_root_checkpoint UNIQUE (log_id, tree_root, tree_size)
);
-- Index for finding latest checkpoints per log
CREATE INDEX IF NOT EXISTS idx_rekor_root_checkpoints_latest
ON attestor.rekor_root_checkpoints(log_id, captured_at DESC);
-- Index for consistency verification
CREATE INDEX IF NOT EXISTS idx_rekor_root_checkpoints_unverified
ON attestor.rekor_root_checkpoints(captured_at DESC)
WHERE verified_at IS NULL;
COMMENT ON TABLE attestor.rekor_root_checkpoints IS 'Stores Rekor tree root checkpoints for consistency verification';
COMMENT ON COLUMN attestor.rekor_root_checkpoints.tree_root IS 'Merkle tree root hash (base64)';
COMMENT ON COLUMN attestor.rekor_root_checkpoints.tree_size IS 'Tree size at checkpoint';
COMMENT ON COLUMN attestor.rekor_root_checkpoints.log_id IS 'Rekor log identifier (hash of public key)';
COMMENT ON COLUMN attestor.rekor_root_checkpoints.checkpoint_envelope IS 'Signed checkpoint in note format';
COMMENT ON COLUMN attestor.rekor_root_checkpoints.is_consistent IS 'Whether checkpoint was consistent with previous';
COMMENT ON COLUMN attestor.rekor_root_checkpoints.inconsistency_reason IS 'Reason for inconsistency if detected';

139
deploy/database/migrations/V20260119_001__Add_UnderReview_Escalated_Rejected_States.sql Normal file
View File

@@ -0,0 +1,139 @@
-- -----------------------------------------------------------------------------
-- V20260119_001__Add_UnderReview_Escalated_Rejected_States.sql
-- Sprint: SPRINT_20260118_018_Unknowns_queue_enhancement
-- Task: UQ-005 - Migration for existing entries (map to new states)
-- Description: Adds new state machine states and required columns
-- -----------------------------------------------------------------------------
-- Add new columns for UnderReview and Escalated states
ALTER TABLE grey_queue_entries
ADD COLUMN IF NOT EXISTS assignee VARCHAR(255) NULL,
ADD COLUMN IF NOT EXISTS assigned_at TIMESTAMPTZ NULL,
ADD COLUMN IF NOT EXISTS escalated_at TIMESTAMPTZ NULL,
ADD COLUMN IF NOT EXISTS escalation_reason TEXT NULL;
-- Add new enum values to grey_queue_status
-- Note: PostgreSQL requires special handling for enum additions
-- First, check if we need to add the values (idempotent)
DO $$
BEGIN
-- Add 'under_review' if not exists
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = 'under_review'
AND enumtypid = 'grey_queue_status'::regtype
) THEN
ALTER TYPE grey_queue_status ADD VALUE 'under_review' AFTER 'retrying';
END IF;
-- Add 'escalated' if not exists
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = 'escalated'
AND enumtypid = 'grey_queue_status'::regtype
) THEN
ALTER TYPE grey_queue_status ADD VALUE 'escalated' AFTER 'under_review';
END IF;
-- Add 'rejected' if not exists
IF NOT EXISTS (
SELECT 1 FROM pg_enum
WHERE enumlabel = 'rejected'
AND enumtypid = 'grey_queue_status'::regtype
) THEN
ALTER TYPE grey_queue_status ADD VALUE 'rejected' AFTER 'resolved';
END IF;
EXCEPTION
WHEN others THEN
-- Enum values may already exist, which is fine
NULL;
END $$;
-- Add indexes for new query patterns
CREATE INDEX IF NOT EXISTS idx_grey_queue_assignee
ON grey_queue_entries(assignee)
WHERE assignee IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_grey_queue_status_assignee
ON grey_queue_entries(status, assignee)
WHERE status IN ('under_review', 'escalated');
CREATE INDEX IF NOT EXISTS idx_grey_queue_escalated_at
ON grey_queue_entries(escalated_at DESC)
WHERE escalated_at IS NOT NULL;
-- Add audit trigger for state transitions
CREATE TABLE IF NOT EXISTS grey_queue_state_transitions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
entry_id UUID NOT NULL REFERENCES grey_queue_entries(id),
tenant_id VARCHAR(128) NOT NULL,
from_state VARCHAR(32) NOT NULL,
to_state VARCHAR(32) NOT NULL,
transitioned_by VARCHAR(255),
reason TEXT,
transitioned_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
metadata JSONB
);
CREATE INDEX IF NOT EXISTS idx_grey_queue_transitions_entry
ON grey_queue_state_transitions(entry_id);
CREATE INDEX IF NOT EXISTS idx_grey_queue_transitions_tenant_time
ON grey_queue_state_transitions(tenant_id, transitioned_at DESC);
-- Function to record state transitions
CREATE OR REPLACE FUNCTION record_grey_queue_transition()
RETURNS TRIGGER AS $$
BEGIN
IF OLD.status IS DISTINCT FROM NEW.status THEN
INSERT INTO grey_queue_state_transitions (
entry_id, tenant_id, from_state, to_state,
transitioned_by, transitioned_at
) VALUES (
NEW.id,
NEW.tenant_id,
OLD.status::text,
NEW.status::text,
COALESCE(NEW.assignee, current_user),
NOW()
);
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Create trigger if not exists
DROP TRIGGER IF EXISTS trg_grey_queue_state_transition ON grey_queue_entries;
CREATE TRIGGER trg_grey_queue_state_transition
AFTER UPDATE ON grey_queue_entries
FOR EACH ROW
EXECUTE FUNCTION record_grey_queue_transition();
-- Update summary view to include new states
CREATE OR REPLACE VIEW grey_queue_summary AS
SELECT
tenant_id,
COUNT(*) FILTER (WHERE status = 'pending') as pending_count,
COUNT(*) FILTER (WHERE status = 'processing') as processing_count,
COUNT(*) FILTER (WHERE status = 'retrying') as retrying_count,
COUNT(*) FILTER (WHERE status = 'under_review') as under_review_count,
COUNT(*) FILTER (WHERE status = 'escalated') as escalated_count,
COUNT(*) FILTER (WHERE status = 'resolved') as resolved_count,
COUNT(*) FILTER (WHERE status = 'rejected') as rejected_count,
COUNT(*) FILTER (WHERE status = 'failed') as failed_count,
COUNT(*) FILTER (WHERE status = 'expired') as expired_count,
COUNT(*) FILTER (WHERE status = 'dismissed') as dismissed_count,
COUNT(*) as total_count
FROM grey_queue_entries
GROUP BY tenant_id;
-- Comment for documentation
COMMENT ON COLUMN grey_queue_entries.assignee IS
'Assignee for entries in UnderReview state (Sprint UQ-005)';
COMMENT ON COLUMN grey_queue_entries.assigned_at IS
'When the entry was assigned for review (Sprint UQ-005)';
COMMENT ON COLUMN grey_queue_entries.escalated_at IS
'When the entry was escalated to security team (Sprint UQ-005)';
COMMENT ON COLUMN grey_queue_entries.escalation_reason IS
'Reason for escalation (Sprint UQ-005)';

130
deploy/database/migrations/V20260119__scanner_layer_diffid.sql Normal file
View File

@@ -0,0 +1,130 @@
-- Migration: Add diff_id column to scanner layers table
-- Sprint: SPRINT_025_Scanner_layer_manifest_infrastructure
-- Task: TASK-025-03
-- Add diff_id column to layers table (sha256:64hex = 71 chars)
ALTER TABLE scanner.layers
ADD COLUMN IF NOT EXISTS diff_id VARCHAR(71);
-- Add timestamp for when diffID was computed
ALTER TABLE scanner.layers
ADD COLUMN IF NOT EXISTS diff_id_computed_at_utc TIMESTAMP;
-- Create index on diff_id for fast lookups
CREATE INDEX IF NOT EXISTS idx_layers_diff_id
ON scanner.layers (diff_id)
WHERE diff_id IS NOT NULL;
-- Create image_layers junction table if it doesn't exist
-- This tracks which layers belong to which images
CREATE TABLE IF NOT EXISTS scanner.image_layers (
image_reference VARCHAR(512) NOT NULL,
layer_digest VARCHAR(71) NOT NULL,
layer_index INT NOT NULL,
created_at_utc TIMESTAMP NOT NULL DEFAULT NOW(),
PRIMARY KEY (image_reference, layer_digest)
);
CREATE INDEX IF NOT EXISTS idx_image_layers_digest
ON scanner.image_layers (layer_digest);
-- DiffID cache table for resolved diffIDs
CREATE TABLE IF NOT EXISTS scanner.scanner_diffid_cache (
layer_digest VARCHAR(71) PRIMARY KEY,
diff_id VARCHAR(71) NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- Base image fingerprint tables for layer reuse detection
CREATE TABLE IF NOT EXISTS scanner.scanner_base_image_fingerprints (
image_reference VARCHAR(512) PRIMARY KEY,
layer_count INT NOT NULL,
registered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
detection_count BIGINT NOT NULL DEFAULT 0
);
CREATE TABLE IF NOT EXISTS scanner.scanner_base_image_layers (
image_reference VARCHAR(512) NOT NULL REFERENCES scanner.scanner_base_image_fingerprints(image_reference) ON DELETE CASCADE,
layer_index INT NOT NULL,
diff_id VARCHAR(71) NOT NULL,
PRIMARY KEY (image_reference, layer_index)
);
CREATE INDEX IF NOT EXISTS idx_base_image_layers_diff_id
ON scanner.scanner_base_image_layers (diff_id);
-- Manifest snapshots table for IOciManifestSnapshotService
CREATE TABLE IF NOT EXISTS scanner.manifest_snapshots (
id UUID PRIMARY KEY,
image_reference VARCHAR(512) NOT NULL,
registry VARCHAR(256) NOT NULL,
repository VARCHAR(256) NOT NULL,
tag VARCHAR(128),
manifest_digest VARCHAR(71) NOT NULL,
config_digest VARCHAR(71) NOT NULL,
media_type VARCHAR(128) NOT NULL,
layers JSONB NOT NULL,
diff_ids JSONB NOT NULL,
platform JSONB,
total_size BIGINT NOT NULL,
captured_at TIMESTAMPTZ NOT NULL,
snapshot_version VARCHAR(32),
UNIQUE (manifest_digest)
);
CREATE INDEX IF NOT EXISTS idx_manifest_snapshots_image_ref
ON scanner.manifest_snapshots (image_reference);
CREATE INDEX IF NOT EXISTS idx_manifest_snapshots_repository
ON scanner.manifest_snapshots (registry, repository);
CREATE INDEX IF NOT EXISTS idx_manifest_snapshots_captured_at
ON scanner.manifest_snapshots (captured_at DESC);
-- Layer scan history for reuse detection (TASK-025-04)
CREATE TABLE IF NOT EXISTS scanner.layer_scans (
diff_id VARCHAR(71) PRIMARY KEY,
scanned_at TIMESTAMPTZ NOT NULL,
finding_count INT,
scanned_by VARCHAR(128) NOT NULL,
scanner_version VARCHAR(64)
);
-- Layer reuse counts for statistics
CREATE TABLE IF NOT EXISTS scanner.layer_reuse_counts (
diff_id VARCHAR(71) PRIMARY KEY,
reuse_count INT NOT NULL DEFAULT 1,
first_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_layer_reuse_counts_count
ON scanner.layer_reuse_counts (reuse_count DESC);
COMMENT ON COLUMN scanner.layers.diff_id IS 'Uncompressed layer content hash (sha256:hex64). Immutable once computed.';
COMMENT ON TABLE scanner.scanner_diffid_cache IS 'Cache of layer digest to diffID mappings. Layer digests are immutable so cache entries never expire.';
COMMENT ON TABLE scanner.scanner_base_image_fingerprints IS 'Known base image fingerprints for layer reuse detection.';
COMMENT ON TABLE scanner.manifest_snapshots IS 'Point-in-time captures of OCI image manifests for delta scanning.';
COMMENT ON TABLE scanner.layer_scans IS 'History of layer scans for deduplication. One entry per diffID.';
COMMENT ON TABLE scanner.layer_reuse_counts IS 'Counts of how many times each layer appears across images.';
-- Layer SBOM CAS for per-layer SBOM storage (TASK-026-02)
CREATE TABLE IF NOT EXISTS scanner.layer_sbom_cas (
diff_id VARCHAR(71) NOT NULL,
format VARCHAR(20) NOT NULL,
content BYTEA NOT NULL,
size_bytes BIGINT NOT NULL,
compressed BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_accessed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (diff_id, format)
);
CREATE INDEX IF NOT EXISTS idx_layer_sbom_cas_last_accessed
ON scanner.layer_sbom_cas (last_accessed_at);
CREATE INDEX IF NOT EXISTS idx_layer_sbom_cas_format
ON scanner.layer_sbom_cas (format);
COMMENT ON TABLE scanner.layer_sbom_cas IS 'Content-addressable storage for per-layer SBOMs. Keyed by diffID (immutable).';
COMMENT ON COLUMN scanner.layer_sbom_cas.content IS 'Compressed (gzip) SBOM content.';
COMMENT ON COLUMN scanner.layer_sbom_cas.last_accessed_at IS 'For TTL-based eviction of cold entries.';

561
deploy/database/postgres-partitioning/001_partition_infrastructure.sql Normal file
View File

@@ -0,0 +1,561 @@
-- Partitioning Infrastructure Migration 001: Foundation
-- Sprint: SPRINT_3422_0001_0001 - Time-Based Partitioning
-- Category: C (infrastructure setup, requires planned maintenance)
--
-- Purpose: Create partition management infrastructure including:
-- - Helper functions for partition creation and maintenance
-- - Utility functions for BRIN index optimization
-- - Partition maintenance scheduling support
--
-- This migration creates the foundation; table conversion is done in separate migrations.
BEGIN;
-- ============================================================================
-- Step 1: Create partition management schema
-- ============================================================================
CREATE SCHEMA IF NOT EXISTS partition_mgmt;
COMMENT ON SCHEMA partition_mgmt IS
'Partition management utilities for time-series tables';
-- ============================================================================
-- Step 2: Managed table registration
-- ============================================================================
CREATE TABLE IF NOT EXISTS partition_mgmt.managed_tables (
schema_name TEXT NOT NULL,
table_name TEXT NOT NULL,
partition_key TEXT NOT NULL,
partition_type TEXT NOT NULL,
retention_months INT NOT NULL DEFAULT 0,
months_ahead INT NOT NULL DEFAULT 3,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (schema_name, table_name)
);
COMMENT ON TABLE partition_mgmt.managed_tables IS
'Tracks partitioned tables with retention and creation settings';
-- ============================================================================
-- Step 3: Partition creation function
-- ============================================================================
-- Creates a new partition for a given table and date range
CREATE OR REPLACE FUNCTION partition_mgmt.create_partition(
p_schema_name TEXT,
p_table_name TEXT,
p_partition_column TEXT,
p_start_date DATE,
p_end_date DATE,
p_partition_suffix TEXT DEFAULT NULL
)
RETURNS TEXT
LANGUAGE plpgsql
AS $$
DECLARE
v_partition_name TEXT;
v_parent_table TEXT;
v_sql TEXT;
BEGIN
v_parent_table := format('%I.%I', p_schema_name, p_table_name);
-- Generate partition name: tablename_YYYY_MM or tablename_YYYY_Q#
IF p_partition_suffix IS NOT NULL THEN
v_partition_name := format('%s_%s', p_table_name, p_partition_suffix);
ELSE
v_partition_name := format('%s_%s', p_table_name, to_char(p_start_date, 'YYYY_MM'));
END IF;
-- Check if partition already exists
IF EXISTS (
SELECT 1 FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE n.nspname = p_schema_name AND c.relname = v_partition_name
) THEN
RAISE NOTICE 'Partition % already exists, skipping', v_partition_name;
RETURN v_partition_name;
END IF;
-- Create partition
v_sql := format(
'CREATE TABLE %I.%I PARTITION OF %s FOR VALUES FROM (%L) TO (%L)',
p_schema_name,
v_partition_name,
v_parent_table,
p_start_date,
p_end_date
);
EXECUTE v_sql;
RAISE NOTICE 'Created partition %.%', p_schema_name, v_partition_name;
RETURN v_partition_name;
END;
$$;
-- ============================================================================
-- Step 4: Monthly partition creation helper
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.create_monthly_partitions(
p_schema_name TEXT,
p_table_name TEXT,
p_partition_column TEXT,
p_start_month DATE,
p_months_ahead INT DEFAULT 3
)
RETURNS SETOF TEXT
LANGUAGE plpgsql
AS $$
DECLARE
v_current_month DATE;
v_end_month DATE;
v_partition_name TEXT;
BEGIN
v_current_month := date_trunc('month', p_start_month)::DATE;
v_end_month := date_trunc('month', NOW() + (p_months_ahead || ' months')::INTERVAL)::DATE;
WHILE v_current_month <= v_end_month LOOP
v_partition_name := partition_mgmt.create_partition(
p_schema_name,
p_table_name,
p_partition_column,
v_current_month,
(v_current_month + INTERVAL '1 month')::DATE
);
RETURN NEXT v_partition_name;
v_current_month := (v_current_month + INTERVAL '1 month')::DATE;
END LOOP;
END;
$$;
-- ============================================================================
-- Step 5: Quarterly partition creation helper
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.create_quarterly_partitions(
p_schema_name TEXT,
p_table_name TEXT,
p_partition_column TEXT,
p_start_quarter DATE,
p_quarters_ahead INT DEFAULT 2
)
RETURNS SETOF TEXT
LANGUAGE plpgsql
AS $$
DECLARE
v_current_quarter DATE;
v_end_quarter DATE;
v_partition_name TEXT;
v_suffix TEXT;
BEGIN
v_current_quarter := date_trunc('quarter', p_start_quarter)::DATE;
v_end_quarter := date_trunc('quarter', NOW() + (p_quarters_ahead * 3 || ' months')::INTERVAL)::DATE;
WHILE v_current_quarter <= v_end_quarter LOOP
-- Generate suffix like 2025_Q1, 2025_Q2, etc.
v_suffix := to_char(v_current_quarter, 'YYYY') || '_Q' ||
EXTRACT(QUARTER FROM v_current_quarter)::TEXT;
v_partition_name := partition_mgmt.create_partition(
p_schema_name,
p_table_name,
p_partition_column,
v_current_quarter,
(v_current_quarter + INTERVAL '3 months')::DATE,
v_suffix
);
RETURN NEXT v_partition_name;
v_current_quarter := (v_current_quarter + INTERVAL '3 months')::DATE;
END LOOP;
END;
$$;
-- ============================================================================
-- Step 6: Ensure future partitions exist
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.ensure_future_partitions(
p_schema_name TEXT,
p_table_name TEXT,
p_months_ahead INT
)
RETURNS INT
LANGUAGE plpgsql
AS $$
DECLARE
v_partition_key TEXT;
v_partition_type TEXT;
v_months_ahead INT;
v_created INT := 0;
v_current DATE;
v_end DATE;
v_suffix TEXT;
v_partition_name TEXT;
BEGIN
SELECT partition_key, partition_type, months_ahead
INTO v_partition_key, v_partition_type, v_months_ahead
FROM partition_mgmt.managed_tables
WHERE schema_name = p_schema_name
AND table_name = p_table_name;
IF v_partition_key IS NULL THEN
RETURN 0;
END IF;
IF p_months_ahead IS NOT NULL AND p_months_ahead > 0 THEN
v_months_ahead := p_months_ahead;
END IF;
IF v_months_ahead IS NULL OR v_months_ahead <= 0 THEN
RETURN 0;
END IF;
v_partition_type := lower(coalesce(v_partition_type, 'monthly'));
IF v_partition_type = 'monthly' THEN
v_current := date_trunc('month', NOW())::DATE;
v_end := date_trunc('month', NOW() + (v_months_ahead || ' months')::INTERVAL)::DATE;
WHILE v_current <= v_end LOOP
v_partition_name := format('%s_%s', p_table_name, to_char(v_current, 'YYYY_MM'));
IF NOT EXISTS (
SELECT 1 FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE n.nspname = p_schema_name AND c.relname = v_partition_name
) THEN
PERFORM partition_mgmt.create_partition(
p_schema_name,
p_table_name,
v_partition_key,
v_current,
(v_current + INTERVAL '1 month')::DATE
);
v_created := v_created + 1;
END IF;
v_current := (v_current + INTERVAL '1 month')::DATE;
END LOOP;
ELSIF v_partition_type = 'quarterly' THEN
v_current := date_trunc('quarter', NOW())::DATE;
v_end := date_trunc('quarter', NOW() + (v_months_ahead || ' months')::INTERVAL)::DATE;
WHILE v_current <= v_end LOOP
v_suffix := to_char(v_current, 'YYYY') || '_Q' ||
EXTRACT(QUARTER FROM v_current)::TEXT;
v_partition_name := format('%s_%s', p_table_name, v_suffix);
IF NOT EXISTS (
SELECT 1 FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE n.nspname = p_schema_name AND c.relname = v_partition_name
) THEN
PERFORM partition_mgmt.create_partition(
p_schema_name,
p_table_name,
v_partition_key,
v_current,
(v_current + INTERVAL '3 months')::DATE,
v_suffix
);
v_created := v_created + 1;
END IF;
v_current := (v_current + INTERVAL '3 months')::DATE;
END LOOP;
END IF;
RETURN v_created;
END;
$$;
-- ============================================================================
-- Step 7: Retention enforcement function
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.enforce_retention(
p_schema_name TEXT,
p_table_name TEXT,
p_retention_months INT
)
RETURNS INT
LANGUAGE plpgsql
AS $$
DECLARE
v_retention_months INT;
v_cutoff_date DATE;
v_partition RECORD;
v_dropped INT := 0;
BEGIN
SELECT retention_months
INTO v_retention_months
FROM partition_mgmt.managed_tables
WHERE schema_name = p_schema_name
AND table_name = p_table_name;
IF p_retention_months IS NOT NULL AND p_retention_months > 0 THEN
v_retention_months := p_retention_months;
END IF;
IF v_retention_months IS NULL OR v_retention_months <= 0 THEN
RETURN 0;
END IF;
v_cutoff_date := (NOW() - (v_retention_months || ' months')::INTERVAL)::DATE;
FOR v_partition IN
SELECT partition_name, partition_end
FROM partition_mgmt.partition_stats
WHERE schema_name = p_schema_name
AND table_name = p_table_name
LOOP
IF v_partition.partition_end IS NOT NULL AND v_partition.partition_end < v_cutoff_date THEN
EXECUTE format('DROP TABLE IF EXISTS %I.%I', p_schema_name, v_partition.partition_name);
v_dropped := v_dropped + 1;
END IF;
END LOOP;
RETURN v_dropped;
END;
$$;
-- ============================================================================
-- Step 8: Partition detach and archive function
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.detach_partition(
p_schema_name TEXT,
p_table_name TEXT,
p_partition_name TEXT,
p_archive_schema TEXT DEFAULT 'archive'
)
RETURNS BOOLEAN
LANGUAGE plpgsql
AS $$
DECLARE
v_parent_table TEXT;
v_partition_full TEXT;
v_archive_table TEXT;
BEGIN
v_parent_table := format('%I.%I', p_schema_name, p_table_name);
v_partition_full := format('%I.%I', p_schema_name, p_partition_name);
v_archive_table := format('%I.%I', p_archive_schema, p_partition_name);
-- Create archive schema if not exists
EXECUTE format('CREATE SCHEMA IF NOT EXISTS %I', p_archive_schema);
-- Detach partition
EXECUTE format(
'ALTER TABLE %s DETACH PARTITION %s',
v_parent_table,
v_partition_full
);
-- Move to archive schema
EXECUTE format(
'ALTER TABLE %s SET SCHEMA %I',
v_partition_full,
p_archive_schema
);
RAISE NOTICE 'Detached and archived partition % to %', p_partition_name, v_archive_table;
RETURN TRUE;
EXCEPTION
WHEN OTHERS THEN
RAISE WARNING 'Failed to detach partition %: %', p_partition_name, SQLERRM;
RETURN FALSE;
END;
$$;
-- ============================================================================
-- Step 9: Partition retention cleanup function
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.cleanup_old_partitions(
p_schema_name TEXT,
p_table_name TEXT,
p_retention_months INT,
p_archive_schema TEXT DEFAULT 'archive',
p_dry_run BOOLEAN DEFAULT TRUE
)
RETURNS TABLE(partition_name TEXT, action TEXT)
LANGUAGE plpgsql
AS $$
DECLARE
v_cutoff_date DATE;
v_partition RECORD;
v_partition_end DATE;
BEGIN
v_cutoff_date := (NOW() - (p_retention_months || ' months')::INTERVAL)::DATE;
FOR v_partition IN
SELECT c.relname as name,
pg_get_expr(c.relpartbound, c.oid) as bound_expr
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE n.nspname = p_schema_name
AND parent.relname = p_table_name
AND c.relkind = 'r'
LOOP
-- Parse the partition bound to get end date
-- Format: FOR VALUES FROM ('2024-01-01') TO ('2024-02-01')
v_partition_end := (regexp_match(v_partition.bound_expr,
'TO \(''([^'']+)''\)'))[1]::DATE;
IF v_partition_end IS NOT NULL AND v_partition_end < v_cutoff_date THEN
partition_name := v_partition.name;
IF p_dry_run THEN
action := 'WOULD_ARCHIVE';
ELSE
IF partition_mgmt.detach_partition(
p_schema_name, p_table_name, v_partition.name, p_archive_schema
) THEN
action := 'ARCHIVED';
ELSE
action := 'FAILED';
END IF;
END IF;
RETURN NEXT;
END IF;
END LOOP;
END;
$$;
-- ============================================================================
-- Step 10: Partition statistics view
-- ============================================================================
CREATE OR REPLACE VIEW partition_mgmt.partition_stats AS
SELECT
n.nspname AS schema_name,
parent.relname AS table_name,
c.relname AS partition_name,
pg_get_expr(c.relpartbound, c.oid) AS partition_range,
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \(''([^'']+)''\)'))[1]::DATE AS partition_start,
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \(''([^'']+)''\)'))[1]::DATE AS partition_end,
pg_size_pretty(pg_relation_size(c.oid)) AS size,
pg_relation_size(c.oid) AS size_bytes,
COALESCE(s.n_live_tup, 0) AS estimated_rows,
s.last_vacuum,
s.last_autovacuum,
s.last_analyze,
s.last_autoanalyze
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
LEFT JOIN pg_stat_user_tables s ON c.oid = s.relid
WHERE c.relkind = 'r'
AND parent.relkind = 'p'
ORDER BY n.nspname, parent.relname, c.relname;
COMMENT ON VIEW partition_mgmt.partition_stats IS
'Statistics for all partitioned tables in the database';
-- ============================================================================
-- Step 11: BRIN index optimization helper
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.create_brin_index_if_not_exists(
p_schema_name TEXT,
p_table_name TEXT,
p_column_name TEXT,
p_pages_per_range INT DEFAULT 128
)
RETURNS BOOLEAN
LANGUAGE plpgsql
AS $$
DECLARE
v_index_name TEXT;
v_sql TEXT;
BEGIN
v_index_name := format('brin_%s_%s', p_table_name, p_column_name);
-- Check if index exists
IF EXISTS (
SELECT 1 FROM pg_indexes
WHERE schemaname = p_schema_name AND indexname = v_index_name
) THEN
RAISE NOTICE 'BRIN index % already exists', v_index_name;
RETURN FALSE;
END IF;
v_sql := format(
'CREATE INDEX %I ON %I.%I USING brin (%I) WITH (pages_per_range = %s)',
v_index_name,
p_schema_name,
p_table_name,
p_column_name,
p_pages_per_range
);
EXECUTE v_sql;
RAISE NOTICE 'Created BRIN index % on %.%(%)',
v_index_name, p_schema_name, p_table_name, p_column_name;
RETURN TRUE;
END;
$$;
-- ============================================================================
-- Step 12: Maintenance job tracking table
-- ============================================================================
CREATE TABLE IF NOT EXISTS partition_mgmt.maintenance_log (
id BIGSERIAL PRIMARY KEY,
operation TEXT NOT NULL,
schema_name TEXT NOT NULL,
table_name TEXT NOT NULL,
partition_name TEXT,
status TEXT NOT NULL DEFAULT 'started',
details JSONB NOT NULL DEFAULT '{}',
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
error_message TEXT
);
CREATE INDEX idx_maintenance_log_table ON partition_mgmt.maintenance_log(schema_name, table_name);
CREATE INDEX idx_maintenance_log_status ON partition_mgmt.maintenance_log(status, started_at);
-- ============================================================================
-- Step 13: Archive schema for detached partitions
-- ============================================================================
CREATE SCHEMA IF NOT EXISTS archive;
COMMENT ON SCHEMA archive IS
'Storage for detached/archived partitions awaiting deletion or offload';
COMMIT;
-- ============================================================================
-- Usage Examples (commented out)
-- ============================================================================
/*
-- Create monthly partitions for audit table, 3 months ahead
SELECT partition_mgmt.create_monthly_partitions(
'scheduler', 'audit', 'created_at', '2024-01-01'::DATE, 3
);
-- Preview old partitions that would be archived (dry run)
SELECT * FROM partition_mgmt.cleanup_old_partitions(
'scheduler', 'audit', 12, 'archive', TRUE
);
-- Actually archive old partitions
SELECT * FROM partition_mgmt.cleanup_old_partitions(
'scheduler', 'audit', 12, 'archive', FALSE
);
-- View partition statistics
SELECT * FROM partition_mgmt.partition_stats
WHERE schema_name = 'scheduler'
ORDER BY table_name, partition_name;
*/

143
deploy/database/postgres-partitioning/002_calibration_schema.sql Normal file
View File

@@ -0,0 +1,143 @@
-- Migration: Trust Vector Calibration Schema
-- Sprint: 7100.0002.0002
-- Description: Creates schema and tables for trust vector calibration system
-- Create calibration schema
CREATE SCHEMA IF NOT EXISTS excititor_calibration;
-- Calibration manifests table
-- Stores signed manifests for each calibration epoch
CREATE TABLE IF NOT EXISTS excititor_calibration.calibration_manifests (
manifest_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
epoch_number INTEGER NOT NULL,
epoch_start_utc TIMESTAMP NOT NULL,
epoch_end_utc TIMESTAMP NOT NULL,
sample_count INTEGER NOT NULL,
learning_rate DOUBLE PRECISION NOT NULL,
policy_hash TEXT,
lattice_version TEXT NOT NULL,
manifest_json JSONB NOT NULL,
signature_envelope JSONB,
created_at_utc TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
created_by TEXT NOT NULL,
CONSTRAINT uq_calibration_manifest_tenant_epoch UNIQUE (tenant_id, epoch_number)
);
CREATE INDEX idx_calibration_manifests_tenant
ON excititor_calibration.calibration_manifests(tenant_id);
CREATE INDEX idx_calibration_manifests_created
ON excititor_calibration.calibration_manifests(created_at_utc DESC);
-- Trust vector adjustments table
-- Records each provider's trust vector changes per epoch
CREATE TABLE IF NOT EXISTS excititor_calibration.trust_vector_adjustments (
adjustment_id BIGSERIAL PRIMARY KEY,
manifest_id TEXT NOT NULL REFERENCES excititor_calibration.calibration_manifests(manifest_id),
source_id TEXT NOT NULL,
old_provenance DOUBLE PRECISION NOT NULL,
old_coverage DOUBLE PRECISION NOT NULL,
old_replayability DOUBLE PRECISION NOT NULL,
new_provenance DOUBLE PRECISION NOT NULL,
new_coverage DOUBLE PRECISION NOT NULL,
new_replayability DOUBLE PRECISION NOT NULL,
adjustment_magnitude DOUBLE PRECISION NOT NULL,
confidence_in_adjustment DOUBLE PRECISION NOT NULL,
sample_count_for_source INTEGER NOT NULL,
created_at_utc TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
CONSTRAINT chk_old_provenance_range CHECK (old_provenance >= 0 AND old_provenance <= 1),
CONSTRAINT chk_old_coverage_range CHECK (old_coverage >= 0 AND old_coverage <= 1),
CONSTRAINT chk_old_replayability_range CHECK (old_replayability >= 0 AND old_replayability <= 1),
CONSTRAINT chk_new_provenance_range CHECK (new_provenance >= 0 AND new_provenance <= 1),
CONSTRAINT chk_new_coverage_range CHECK (new_coverage >= 0 AND new_coverage <= 1),
CONSTRAINT chk_new_replayability_range CHECK (new_replayability >= 0 AND new_replayability <= 1),
CONSTRAINT chk_confidence_range CHECK (confidence_in_adjustment >= 0 AND confidence_in_adjustment <= 1)
);
CREATE INDEX idx_trust_adjustments_manifest
ON excititor_calibration.trust_vector_adjustments(manifest_id);
CREATE INDEX idx_trust_adjustments_source
ON excititor_calibration.trust_vector_adjustments(source_id);
-- Calibration feedback samples table
-- Stores empirical evidence used for calibration
CREATE TABLE IF NOT EXISTS excititor_calibration.calibration_samples (
sample_id BIGSERIAL PRIMARY KEY,
tenant_id TEXT NOT NULL,
source_id TEXT NOT NULL,
cve_id TEXT NOT NULL,
purl TEXT NOT NULL,
expected_status TEXT NOT NULL,
actual_status TEXT NOT NULL,
verdict_confidence DOUBLE PRECISION NOT NULL,
is_match BOOLEAN NOT NULL,
feedback_source TEXT NOT NULL, -- 'reachability', 'customer_feedback', 'integration_tests'
feedback_weight DOUBLE PRECISION NOT NULL DEFAULT 1.0,
scan_id TEXT,
collected_at_utc TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
processed BOOLEAN NOT NULL DEFAULT FALSE,
processed_in_manifest_id TEXT REFERENCES excititor_calibration.calibration_manifests(manifest_id),
CONSTRAINT chk_verdict_confidence_range CHECK (verdict_confidence >= 0 AND verdict_confidence <= 1),
CONSTRAINT chk_feedback_weight_range CHECK (feedback_weight >= 0 AND feedback_weight <= 1)
);
CREATE INDEX idx_calibration_samples_tenant
ON excititor_calibration.calibration_samples(tenant_id);
CREATE INDEX idx_calibration_samples_source
ON excititor_calibration.calibration_samples(source_id);
CREATE INDEX idx_calibration_samples_collected
ON excititor_calibration.calibration_samples(collected_at_utc DESC);
CREATE INDEX idx_calibration_samples_processed
ON excititor_calibration.calibration_samples(processed) WHERE NOT processed;
-- Calibration metrics table
-- Tracks performance metrics per source/severity/status
CREATE TABLE IF NOT EXISTS excititor_calibration.calibration_metrics (
metric_id BIGSERIAL PRIMARY KEY,
manifest_id TEXT NOT NULL REFERENCES excititor_calibration.calibration_manifests(manifest_id),
source_id TEXT,
severity TEXT,
status TEXT,
precision DOUBLE PRECISION NOT NULL,
recall DOUBLE PRECISION NOT NULL,
f1_score DOUBLE PRECISION NOT NULL,
false_positive_rate DOUBLE PRECISION NOT NULL,
false_negative_rate DOUBLE PRECISION NOT NULL,
sample_count INTEGER NOT NULL,
created_at_utc TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
CONSTRAINT chk_precision_range CHECK (precision >= 0 AND precision <= 1),
CONSTRAINT chk_recall_range CHECK (recall >= 0 AND recall <= 1),
CONSTRAINT chk_f1_range CHECK (f1_score >= 0 AND f1_score <= 1),
CONSTRAINT chk_fpr_range CHECK (false_positive_rate >= 0 AND false_positive_rate <= 1),
CONSTRAINT chk_fnr_range CHECK (false_negative_rate >= 0 AND false_negative_rate <= 1)
);
CREATE INDEX idx_calibration_metrics_manifest
ON excititor_calibration.calibration_metrics(manifest_id);
CREATE INDEX idx_calibration_metrics_source
ON excititor_calibration.calibration_metrics(source_id) WHERE source_id IS NOT NULL;
-- Grant permissions to excititor service role
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'excititor_service') THEN
GRANT USAGE ON SCHEMA excititor_calibration TO excititor_service;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA excititor_calibration TO excititor_service;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA excititor_calibration TO excititor_service;
ALTER DEFAULT PRIVILEGES IN SCHEMA excititor_calibration
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO excititor_service;
ALTER DEFAULT PRIVILEGES IN SCHEMA excititor_calibration
GRANT USAGE, SELECT ON SEQUENCES TO excititor_service;
END IF;
END $$;
-- Comments for documentation
COMMENT ON SCHEMA excititor_calibration IS 'Trust vector calibration data for VEX source scoring';
COMMENT ON TABLE excititor_calibration.calibration_manifests IS 'Signed calibration epoch results';
COMMENT ON TABLE excititor_calibration.trust_vector_adjustments IS 'Per-source trust vector changes per epoch';
COMMENT ON TABLE excititor_calibration.calibration_samples IS 'Empirical feedback samples for calibration';
COMMENT ON TABLE excititor_calibration.calibration_metrics IS 'Performance metrics per calibration epoch';

97
deploy/database/postgres-partitioning/provcache/create_provcache_schema.sql Normal file
View File

@@ -0,0 +1,97 @@
-- Provcache schema migration
-- Run as: psql -d stellaops -f create_provcache_schema.sql
-- Create schema
CREATE SCHEMA IF NOT EXISTS provcache;
-- Main cache items table
CREATE TABLE IF NOT EXISTS provcache.provcache_items (
verikey TEXT PRIMARY KEY,
digest_version TEXT NOT NULL DEFAULT 'v1',
verdict_hash TEXT NOT NULL,
proof_root TEXT NOT NULL,
replay_seed JSONB NOT NULL,
policy_hash TEXT NOT NULL,
signer_set_hash TEXT NOT NULL,
feed_epoch TEXT NOT NULL,
trust_score INTEGER NOT NULL CHECK (trust_score >= 0 AND trust_score <= 100),
hit_count BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_accessed_at TIMESTAMPTZ,
-- Constraint: expires_at must be after created_at
CONSTRAINT provcache_items_expires_check CHECK (expires_at > created_at)
);
-- Indexes for invalidation queries
CREATE INDEX IF NOT EXISTS idx_provcache_policy_hash
ON provcache.provcache_items(policy_hash);
CREATE INDEX IF NOT EXISTS idx_provcache_signer_set_hash
ON provcache.provcache_items(signer_set_hash);
CREATE INDEX IF NOT EXISTS idx_provcache_feed_epoch
ON provcache.provcache_items(feed_epoch);
CREATE INDEX IF NOT EXISTS idx_provcache_expires_at
ON provcache.provcache_items(expires_at);
CREATE INDEX IF NOT EXISTS idx_provcache_created_at
ON provcache.provcache_items(created_at);
-- Evidence chunks table for large evidence storage
CREATE TABLE IF NOT EXISTS provcache.prov_evidence_chunks (
chunk_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
proof_root TEXT NOT NULL,
chunk_index INTEGER NOT NULL,
chunk_hash TEXT NOT NULL,
blob BYTEA NOT NULL,
blob_size INTEGER NOT NULL,
content_type TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT prov_evidence_chunks_unique_index
UNIQUE (proof_root, chunk_index)
);
CREATE INDEX IF NOT EXISTS idx_prov_chunks_proof_root
ON provcache.prov_evidence_chunks(proof_root);
-- Revocation audit log
CREATE TABLE IF NOT EXISTS provcache.prov_revocations (
revocation_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
revocation_type TEXT NOT NULL,
target_hash TEXT NOT NULL,
reason TEXT,
actor TEXT,
entries_affected BIGINT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_prov_revocations_created_at
ON provcache.prov_revocations(created_at);
CREATE INDEX IF NOT EXISTS idx_prov_revocations_target_hash
ON provcache.prov_revocations(target_hash);
-- Function to update updated_at timestamp
CREATE OR REPLACE FUNCTION provcache.update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ language 'plpgsql';
-- Trigger for auto-updating updated_at
DROP TRIGGER IF EXISTS update_provcache_items_updated_at ON provcache.provcache_items;
CREATE TRIGGER update_provcache_items_updated_at
BEFORE UPDATE ON provcache.provcache_items
FOR EACH ROW
EXECUTE FUNCTION provcache.update_updated_at_column();
-- Grant permissions (adjust role as needed)
-- GRANT USAGE ON SCHEMA provcache TO stellaops_app;
-- GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA provcache TO stellaops_app;
-- GRANT USAGE ON ALL SEQUENCES IN SCHEMA provcache TO stellaops_app;
COMMENT ON TABLE provcache.provcache_items IS 'Provenance cache entries for cached security decisions';
COMMENT ON TABLE provcache.prov_evidence_chunks IS 'Chunked evidence storage for large SBOMs and attestations';
COMMENT ON TABLE provcache.prov_revocations IS 'Audit log of cache invalidation events';

159
deploy/database/postgres-validation/001_validate_rls.sql Normal file
View File

@@ -0,0 +1,159 @@
-- RLS Validation Script
-- Sprint: SPRINT_3421_0001_0001 - RLS Expansion
--
-- Purpose: Verify that RLS is properly configured on all tenant-scoped tables
-- Run this script after deploying RLS migrations to validate configuration
-- ============================================================================
-- Part 1: List all tables with RLS status
-- ============================================================================
\echo '=== RLS Status for All Schemas ==='
SELECT
schemaname AS schema,
tablename AS table_name,
rowsecurity AS rls_enabled,
forcerowsecurity AS rls_forced,
CASE
WHEN rowsecurity AND forcerowsecurity THEN 'OK'
WHEN rowsecurity AND NOT forcerowsecurity THEN 'WARN: Not forced'
ELSE 'MISSING'
END AS status
FROM pg_tables
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
ORDER BY schemaname, tablename;
-- ============================================================================
-- Part 2: List all RLS policies
-- ============================================================================
\echo ''
\echo '=== RLS Policies ==='
SELECT
schemaname AS schema,
tablename AS table_name,
policyname AS policy_name,
permissive,
roles,
cmd AS applies_to,
qual IS NOT NULL AS has_using,
with_check IS NOT NULL AS has_check
FROM pg_policies
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
ORDER BY schemaname, tablename, policyname;
-- ============================================================================
-- Part 3: Tables missing RLS that should have it (have tenant_id column)
-- ============================================================================
\echo ''
\echo '=== Tables with tenant_id but NO RLS ==='
SELECT
c.table_schema AS schema,
c.table_name AS table_name,
'MISSING RLS' AS issue
FROM information_schema.columns c
JOIN pg_tables t ON c.table_schema = t.schemaname AND c.table_name = t.tablename
WHERE c.column_name IN ('tenant_id', 'tenant')
AND c.table_schema IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
AND NOT t.rowsecurity
ORDER BY c.table_schema, c.table_name;
-- ============================================================================
-- Part 4: Verify helper functions exist
-- ============================================================================
\echo ''
\echo '=== RLS Helper Functions ==='
SELECT
n.nspname AS schema,
p.proname AS function_name,
CASE
WHEN p.prosecdef THEN 'SECURITY DEFINER'
ELSE 'SECURITY INVOKER'
END AS security,
CASE
WHEN p.provolatile = 's' THEN 'STABLE'
WHEN p.provolatile = 'i' THEN 'IMMUTABLE'
ELSE 'VOLATILE'
END AS volatility
FROM pg_proc p
JOIN pg_namespace n ON p.pronamespace = n.oid
WHERE p.proname = 'require_current_tenant'
AND n.nspname LIKE '%_app'
ORDER BY n.nspname;
-- ============================================================================
-- Part 5: Test RLS enforcement (expect failure without tenant context)
-- ============================================================================
\echo ''
\echo '=== RLS Enforcement Test ==='
\echo 'Testing RLS on scheduler.runs (should fail without tenant context)...'
-- Reset tenant context
SELECT set_config('app.tenant_id', '', false);
DO $$
BEGIN
-- This should raise an exception if RLS is working
PERFORM * FROM scheduler.runs LIMIT 1;
RAISE NOTICE 'WARNING: Query succeeded without tenant context - RLS may not be working!';
EXCEPTION
WHEN OTHERS THEN
RAISE NOTICE 'OK: RLS blocked query without tenant context: %', SQLERRM;
END
$$;
-- ============================================================================
-- Part 6: Admin bypass role verification
-- ============================================================================
\echo ''
\echo '=== Admin Bypass Roles ==='
SELECT
rolname AS role_name,
rolbypassrls AS can_bypass_rls,
rolcanlogin AS can_login
FROM pg_roles
WHERE rolname LIKE '%_admin'
AND rolbypassrls = TRUE
ORDER BY rolname;
-- ============================================================================
-- Summary
-- ============================================================================
\echo ''
\echo '=== Summary ==='
SELECT
'Total Tables' AS metric,
COUNT(*)::TEXT AS value
FROM pg_tables
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
UNION ALL
SELECT
'Tables with RLS Enabled',
COUNT(*)::TEXT
FROM pg_tables
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
AND rowsecurity = TRUE
UNION ALL
SELECT
'Tables with RLS Forced',
COUNT(*)::TEXT
FROM pg_tables
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
AND forcerowsecurity = TRUE
UNION ALL
SELECT
'Active Policies',
COUNT(*)::TEXT
FROM pg_policies
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns');

238
deploy/database/postgres-validation/002_validate_partitions.sql Normal file
View File

@@ -0,0 +1,238 @@
-- Partition Validation Script
-- Sprint: SPRINT_3422_0001_0001 - Time-Based Partitioning
--
-- Purpose: Verify that partitioned tables are properly configured and healthy
-- ============================================================================
-- Part 1: List all partitioned tables
-- ============================================================================
\echo '=== Partitioned Tables ==='
SELECT
n.nspname AS schema,
c.relname AS table_name,
CASE pt.partstrat
WHEN 'r' THEN 'RANGE'
WHEN 'l' THEN 'LIST'
WHEN 'h' THEN 'HASH'
END AS partition_strategy,
array_to_string(array_agg(a.attname ORDER BY k.col), ', ') AS partition_key
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_partitioned_table pt ON c.oid = pt.partrelid
JOIN LATERAL unnest(pt.partattrs) WITH ORDINALITY AS k(col, idx) ON true
LEFT JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.col
WHERE n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
GROUP BY n.nspname, c.relname, pt.partstrat
ORDER BY n.nspname, c.relname;
-- ============================================================================
-- Part 2: Partition inventory with sizes
-- ============================================================================
\echo ''
\echo '=== Partition Inventory ==='
SELECT
n.nspname AS schema,
parent.relname AS parent_table,
c.relname AS partition_name,
pg_get_expr(c.relpartbound, c.oid) AS bounds,
pg_size_pretty(pg_relation_size(c.oid)) AS size,
s.n_live_tup AS estimated_rows
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
LEFT JOIN pg_stat_user_tables s ON c.oid = s.relid
WHERE n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
AND c.relkind = 'r'
AND parent.relkind = 'p'
ORDER BY n.nspname, parent.relname, c.relname;
-- ============================================================================
-- Part 3: Check for missing future partitions
-- ============================================================================
\echo ''
\echo '=== Future Partition Coverage ==='
WITH partition_bounds AS (
SELECT
n.nspname AS schema_name,
parent.relname AS table_name,
c.relname AS partition_name,
-- Extract the TO date from partition bound
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \(''([^'']+)''\)'))[1]::DATE AS end_date
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE c.relkind = 'r'
AND parent.relkind = 'p'
AND c.relname NOT LIKE '%_default'
),
max_bounds AS (
SELECT
schema_name,
table_name,
MAX(end_date) AS max_partition_date
FROM partition_bounds
WHERE end_date IS NOT NULL
GROUP BY schema_name, table_name
)
SELECT
schema_name,
table_name,
max_partition_date,
(max_partition_date - CURRENT_DATE) AS days_ahead,
CASE
WHEN (max_partition_date - CURRENT_DATE) < 30 THEN 'CRITICAL: Create partitions!'
WHEN (max_partition_date - CURRENT_DATE) < 60 THEN 'WARNING: Running low'
ELSE 'OK'
END AS status
FROM max_bounds
ORDER BY days_ahead;
-- ============================================================================
-- Part 4: Check for orphaned data in default partitions
-- ============================================================================
\echo ''
\echo '=== Default Partition Data (should be empty) ==='
DO $$
DECLARE
v_schema TEXT;
v_table TEXT;
v_count BIGINT;
v_sql TEXT;
BEGIN
FOR v_schema, v_table IN
SELECT n.nspname, c.relname
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE c.relname LIKE '%_default'
AND n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
LOOP
v_sql := format('SELECT COUNT(*) FROM %I.%I', v_schema, v_table);
EXECUTE v_sql INTO v_count;
IF v_count > 0 THEN
RAISE NOTICE 'WARNING: %.% has % rows in default partition!',
v_schema, v_table, v_count;
ELSE
RAISE NOTICE 'OK: %.% is empty', v_schema, v_table;
END IF;
END LOOP;
END
$$;
-- ============================================================================
-- Part 5: Index health on partitions
-- ============================================================================
\echo ''
\echo '=== Partition Index Coverage ==='
SELECT
schemaname AS schema,
tablename AS table_name,
indexname AS index_name,
indexdef
FROM pg_indexes
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
AND tablename LIKE '%_partitioned' OR tablename LIKE '%_202%'
ORDER BY schemaname, tablename, indexname;
-- ============================================================================
-- Part 6: BRIN index effectiveness check
-- ============================================================================
\echo ''
\echo '=== BRIN Index Statistics ==='
SELECT
schemaname AS schema,
tablename AS table_name,
indexrelname AS index_name,
idx_scan AS scans,
idx_tup_read AS tuples_read,
idx_tup_fetch AS tuples_fetched,
pg_size_pretty(pg_relation_size(indexrelid)) AS index_size
FROM pg_stat_user_indexes
WHERE indexrelname LIKE 'brin_%'
ORDER BY schemaname, tablename;
-- ============================================================================
-- Part 7: Partition maintenance recommendations
-- ============================================================================
\echo ''
\echo '=== Maintenance Recommendations ==='
WITH partition_ages AS (
SELECT
n.nspname AS schema_name,
parent.relname AS table_name,
c.relname AS partition_name,
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \(''([^'']+)''\)'))[1]::DATE AS start_date,
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \(''([^'']+)''\)'))[1]::DATE AS end_date
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE c.relkind = 'r'
AND parent.relkind = 'p'
AND c.relname NOT LIKE '%_default'
)
SELECT
schema_name,
table_name,
partition_name,
start_date,
end_date,
(CURRENT_DATE - end_date) AS days_old,
CASE
WHEN (CURRENT_DATE - end_date) > 365 THEN 'Consider archiving (>1 year old)'
WHEN (CURRENT_DATE - end_date) > 180 THEN 'Review retention policy (>6 months old)'
ELSE 'Current'
END AS recommendation
FROM partition_ages
WHERE start_date IS NOT NULL
ORDER BY schema_name, table_name, start_date;
-- ============================================================================
-- Summary
-- ============================================================================
\echo ''
\echo '=== Summary ==='
SELECT
'Partitioned Tables' AS metric,
COUNT(DISTINCT parent.relname)::TEXT AS value
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
AND parent.relkind = 'p'
UNION ALL
SELECT
'Total Partitions',
COUNT(*)::TEXT
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
AND parent.relkind = 'p'
UNION ALL
SELECT
'BRIN Indexes',
COUNT(*)::TEXT
FROM pg_indexes
WHERE indexname LIKE 'brin_%'
AND schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln');

66
deploy/database/postgres/README.md Normal file
View File

@@ -0,0 +1,66 @@
# PostgreSQL 16 Cluster (staging / production)
This directory provisions StellaOps PostgreSQL clusters with **CloudNativePG (CNPG)**. It is pinned to Postgres 16.x, includes connection pooling (PgBouncer), Prometheus scraping, and S3-compatible backups. Everything is air-gap friendly: fetch the operator and images once, then render/apply manifests offline.
## Targets
- **Staging:** `stellaops-pg-stg` (2 instances, 200 Gi data, WAL 64 Gi, PgBouncer x2)
- **Production:** `stellaops-pg-prod` (3 instances, 500 Gi data, WAL 128 Gi, PgBouncer x3)
- **Namespace:** `platform-postgres`
## Prerequisites
- Kubernetes ≥ 1.27 with CSI storage classes `fast-ssd` (data) and `fast-wal` (WAL) available.
- CloudNativePG operator 1.23.x mirrored or downloaded to `artifacts/cloudnative-pg-1.23.0.yaml`.
- Images mirrored to your registry (example tags):
- `ghcr.io/cloudnative-pg/postgresql:16.4`
- `ghcr.io/cloudnative-pg/postgresql-operator:1.23.0`
- `ghcr.io/cloudnative-pg/pgbouncer:1.23.0`
- Secrets created from the templates under `ops/devops/postgres/secrets/` (superuser, app user, backup credentials).
## Render & Apply (deterministic)
```bash
# 1) Create namespace
kubectl apply -f ops/devops/postgres/namespace.yaml
# 2) Install operator (offline-friendly: use the pinned manifest you mirrored)
kubectl apply -f artifacts/cloudnative-pg-1.23.0.yaml
# 3) Create secrets (replace passwords/keys first)
kubectl apply -f ops/devops/postgres/secrets/example-superuser.yaml
kubectl apply -f ops/devops/postgres/secrets/example-app.yaml
kubectl apply -f ops/devops/postgres/secrets/example-backup-credentials.yaml
# 4) Apply the cluster and pooler for the target environment
kubectl apply -f ops/devops/postgres/cluster-staging.yaml
kubectl apply -f ops/devops/postgres/pooler-staging.yaml
# or
kubectl apply -f ops/devops/postgres/cluster-production.yaml
kubectl apply -f ops/devops/postgres/pooler-production.yaml
```
## Connection Endpoints
- RW service: `<cluster>-rw` (e.g., `stellaops-pg-stg-rw:5432`)
- RO service: `<cluster>-ro`
- PgBouncer pooler: `<pooler-name>` (e.g., `stellaops-pg-stg-pooler:6432`)
**Application connection string (matches library defaults):**
`Host=stellaops-pg-stg-pooler;Port=6432;Username=stellaops_app;Password=<app-password>;Database=stellaops;Pooling=true;Timeout=15;CommandTimeout=30;Ssl Mode=Require;`
## Monitoring & Backups
- `monitoring.enablePodMonitor: true` exposes PodMonitor for Prometheus Operator.
- Barman/S3 backups are enabled by default; set `backup.barmanObjectStore.destinationPath` per env and populate `stellaops-pg-backup` credentials.
- WAL compression is `gzip`; retention is operator-managed (configure via Barman bucket policies).
## Alignment with code defaults
- Session settings: UTC timezone, 30s `statement_timeout`, tenant context via `set_config('app.current_tenant', ...)`.
- Connection pooler uses **transaction** mode with a `server_reset_query` that clears session state, keeping RepositoryBase deterministic.
## Verification checklist
- `kubectl get cluster -n platform-postgres` shows `Ready` replicas matching `instances`.
- `kubectl logs deploy/cnpg-controller-manager -n cnpg-system` has no failing webhooks.
- `kubectl get podmonitor -n platform-postgres` returns entries for the cluster and pooler.
- `psql "<rw-connection-string>" -c 'select 1'` works from CI runner subnet.
- `cnpg` `barman-cloud-backup-list` shows successful full + WAL backups.
## Offline notes
- Mirror the operator manifest and container images to the approved registry first; no live downloads occur at runtime.
- If Prometheus is not present, leave PodMonitor applied; it is inert without the CRD.

57
deploy/database/postgres/cluster-production.yaml Normal file
View File

@@ -0,0 +1,57 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: stellaops-pg-prod
namespace: platform-postgres
spec:
instances: 3
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
primaryUpdateStrategy: unsupervised
storage:
size: 500Gi
storageClass: fast-ssd
walStorage:
size: 128Gi
storageClass: fast-wal
superuserSecret:
name: stellaops-pg-superuser
bootstrap:
initdb:
database: stellaops
owner: stellaops_app
secret:
name: stellaops-pg-app
monitoring:
enablePodMonitor: true
postgresql:
parameters:
max_connections: "900"
shared_buffers: "4096MB"
work_mem: "96MB"
maintenance_work_mem: "768MB"
wal_level: "replica"
max_wal_size: "4GB"
timezone: "UTC"
log_min_duration_statement: "250"
statement_timeout: "30000"
resources:
requests:
cpu: "4"
memory: "16Gi"
limits:
cpu: "8"
memory: "24Gi"
backup:
barmanObjectStore:
destinationPath: s3://stellaops-backups/production
s3Credentials:
accessKeyId:
name: stellaops-pg-backup
key: ACCESS_KEY_ID
secretAccessKey:
name: stellaops-pg-backup
key: SECRET_ACCESS_KEY
wal:
compression: gzip
maxParallel: 4
logLevel: info

57
deploy/database/postgres/cluster-staging.yaml Normal file
View File

@@ -0,0 +1,57 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: stellaops-pg-stg
namespace: platform-postgres
spec:
instances: 2
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
primaryUpdateStrategy: unsupervised
storage:
size: 200Gi
storageClass: fast-ssd
walStorage:
size: 64Gi
storageClass: fast-wal
superuserSecret:
name: stellaops-pg-superuser
bootstrap:
initdb:
database: stellaops
owner: stellaops_app
secret:
name: stellaops-pg-app
monitoring:
enablePodMonitor: true
postgresql:
parameters:
max_connections: "600"
shared_buffers: "2048MB"
work_mem: "64MB"
maintenance_work_mem: "512MB"
wal_level: "replica"
max_wal_size: "2GB"
timezone: "UTC"
log_min_duration_statement: "500"
statement_timeout: "30000"
resources:
requests:
cpu: "2"
memory: "8Gi"
limits:
cpu: "4"
memory: "12Gi"
backup:
barmanObjectStore:
destinationPath: s3://stellaops-backups/staging
s3Credentials:
accessKeyId:
name: stellaops-pg-backup
key: ACCESS_KEY_ID
secretAccessKey:
name: stellaops-pg-backup
key: SECRET_ACCESS_KEY
wal:
compression: gzip
maxParallel: 2
logLevel: info

4
deploy/database/postgres/namespace.yaml Normal file
View File

@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: platform-postgres

29
deploy/database/postgres/pooler-production.yaml Normal file
View File

@@ -0,0 +1,29 @@
apiVersion: postgresql.cnpg.io/v1
kind: Pooler
metadata:
name: stellaops-pg-prod-pooler
namespace: platform-postgres
spec:
cluster:
name: stellaops-pg-prod
instances: 3
type: rw
pgbouncer:
parameters:
pool_mode: transaction
max_client_conn: "1500"
default_pool_size: "80"
server_reset_query: "RESET ALL; SET SESSION AUTHORIZATION DEFAULT; SET TIME ZONE 'UTC';"
authQuerySecret:
name: stellaops-pg-app
template:
spec:
containers:
- name: pgbouncer
resources:
requests:
cpu: "150m"
memory: "192Mi"
limits:
cpu: "750m"
memory: "384Mi"

29
deploy/database/postgres/pooler-staging.yaml Normal file
View File

@@ -0,0 +1,29 @@
apiVersion: postgresql.cnpg.io/v1
kind: Pooler
metadata:
name: stellaops-pg-stg-pooler
namespace: platform-postgres
spec:
cluster:
name: stellaops-pg-stg
instances: 2
type: rw
pgbouncer:
parameters:
pool_mode: transaction
max_client_conn: "800"
default_pool_size: "50"
server_reset_query: "RESET ALL; SET SESSION AUTHORIZATION DEFAULT; SET TIME ZONE 'UTC';"
authQuerySecret:
name: stellaops-pg-app
template:
spec:
containers:
- name: pgbouncer
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"

9
deploy/database/postgres/secrets/example-app.yaml Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: stellaops-pg-app
namespace: platform-postgres
type: kubernetes.io/basic-auth
stringData:
username: stellaops_app
password: CHANGEME_APP_PASSWORD

9
deploy/database/postgres/secrets/example-backup-credentials.yaml Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: stellaops-pg-backup
namespace: platform-postgres
type: Opaque
stringData:
ACCESS_KEY_ID: CHANGEME_ACCESS_KEY
SECRET_ACCESS_KEY: CHANGEME_SECRET_KEY

9
deploy/database/postgres/secrets/example-superuser.yaml Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: stellaops-pg-superuser
namespace: platform-postgres
type: kubernetes.io/basic-auth
stringData:
username: postgres
password: CHANGEME_SUPERUSER_PASSWORD