stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

CD/CD consolidation

Browse Source
This commit is contained in:
StellaOps Bot
2025-12-26 17:32:23 +02:00
parent a866eb6277
commit c786faae84
638 changed files with 3821 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
devops/database/local-postgres/docker-compose.yml Normal file
View File

@@ -0,0 +1,31 @@
version: "3.9"
services:
stella-postgres:
image: postgres:17
container_name: stella-postgres
restart: unless-stopped
environment:
POSTGRES_USER: stella
POSTGRES_PASSWORD: stella
POSTGRES_DB: stella
ports:
- "5432:5432"
volumes:
- stella-postgres-data:/var/lib/postgresql/data
- ./init:/docker-entrypoint-initdb.d:ro
command:
- "postgres"
- "-c"
- "shared_preload_libraries=pg_stat_statements"
- "-c"
- "pg_stat_statements.track=all"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER"]
interval: 10s
timeout: 5s
retries: 5
volumes:
stella-postgres-data:
driver: local

17
devops/database/local-postgres/init/01-extensions.sql Normal file
View File

@@ -0,0 +1,17 @@
-- Enable pg_stat_statements extension for query performance analysis
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
-- Enable other useful extensions
CREATE EXTENSION IF NOT EXISTS pg_trgm; -- Fuzzy text search
CREATE EXTENSION IF NOT EXISTS btree_gin; -- GIN indexes for scalar types
CREATE EXTENSION IF NOT EXISTS pgcrypto; -- Cryptographic functions
-- Create schemas for all modules
CREATE SCHEMA IF NOT EXISTS authority;
CREATE SCHEMA IF NOT EXISTS vuln;
CREATE SCHEMA IF NOT EXISTS vex;
CREATE SCHEMA IF NOT EXISTS scheduler;
CREATE SCHEMA IF NOT EXISTS notify;
CREATE SCHEMA IF NOT EXISTS policy;
CREATE SCHEMA IF NOT EXISTS concelier;
CREATE SCHEMA IF NOT EXISTS audit;

16
devops/database/mongo/indices/README.md Normal file
View File

@@ -0,0 +1,16 @@
# MongoDB Provenance Indexes
Indexes supporting Sprint 401 reachability/provenance queries.
## Available indexes
- `events_by_subject_kind_provenance`: `(subject.digest.sha256, kind, provenance.dsse.rekor.logIndex)` for subject/kind lookups with Rekor presence.
- `events_unproven_by_kind`: `(kind, trust.verified, provenance.dsse.rekor.logIndex)` to find unverified or missing-Rekor events per kind.
- `events_by_rekor_logindex`: `(provenance.dsse.rekor.logIndex)` to audit Rekor alignment.
## Apply
```js
// From mongo shell (connected to provenance database)
load('ops/mongo/indices/events_provenance_indices.js');
```
Indexes are idempotent; rerunning is safe.

89
devops/database/mongo/indices/events_provenance_indices.js Normal file
View File

@@ -0,0 +1,89 @@
/**
* MongoDB indexes for DSSE provenance queries on the events collection.
* Run with: mongosh stellaops_db < events_provenance_indices.js
*
* These indexes support:
* - Proven VEX/SBOM/SCAN lookup by subject digest
* - Compliance gap queries (unverified events)
* - Rekor log index lookups
* - Backfill service queries
*
* Created: 2025-11-27 (PROV-INDEX-401-030)
* C# equivalent: src/StellaOps.Events.Mongo/MongoIndexes.cs
*/
// Switch to the target database (override via --eval "var dbName='custom'" if needed)
const targetDb = typeof dbName !== 'undefined' ? dbName : 'stellaops';
db = db.getSiblingDB(targetDb);
print(`Creating provenance indexes on ${targetDb}.events...`);
// Index 1: Lookup proven events by subject digest + kind
db.events.createIndex(
{
"subject.digest.sha256": 1,
"kind": 1,
"provenance.dsse.rekor.logIndex": 1
},
{
name: "events_by_subject_kind_provenance",
background: true
}
);
print(" - events_by_subject_kind_provenance");
// Index 2: Find unproven evidence by kind (compliance gap queries)
db.events.createIndex(
{
"kind": 1,
"trust.verified": 1,
"provenance.dsse.rekor.logIndex": 1
},
{
name: "events_unproven_by_kind",
background: true
}
);
print(" - events_unproven_by_kind");
// Index 3: Direct Rekor log index lookup
db.events.createIndex(
{
"provenance.dsse.rekor.logIndex": 1
},
{
name: "events_by_rekor_logindex",
background: true
}
);
print(" - events_by_rekor_logindex");
// Index 4: Envelope digest lookup (for backfill deduplication)
db.events.createIndex(
{
"provenance.dsse.envelopeDigest": 1
},
{
name: "events_by_envelope_digest",
background: true,
sparse: true
}
);
print(" - events_by_envelope_digest");
// Index 5: Timestamp + kind for compliance reporting time ranges
db.events.createIndex(
{
"ts": -1,
"kind": 1,
"trust.verified": 1
},
{
name: "events_by_ts_kind_verified",
background: true
}
);
print(" - events_by_ts_kind_verified");
print("\nProvenance indexes created successfully.");
print("Run 'db.events.getIndexes()' to verify.");

67
devops/database/mongo/indices/reachability_store_indices.js Normal file
View File

@@ -0,0 +1,67 @@
/**
* MongoDB indexes for the shared reachability store collections used by Signals/Policy/Scanner.
* Run with: mongosh stellaops_db < reachability_store_indices.js
*
* Collections:
* - func_nodes: canonical function nodes keyed by graph + symbol ID and joinable by (purl, symbolDigest)
* - call_edges: canonical call edges keyed by graph and joinable by (purl, symbolDigest)
* - cve_func_hits: per-subject mapping of CVE -> affected/reachable functions with evidence pointers
*
* Created: 2025-12-13 (SIG-STORE-401-016)
*/
// Switch to the target database (override via --eval "var dbName='custom'" if needed)
const targetDb = typeof dbName !== 'undefined' ? dbName : 'stellaops';
db = db.getSiblingDB(targetDb);
print(`Creating reachability store indexes on ${targetDb}...`);
print(`- func_nodes`);
db.func_nodes.createIndex(
{ "graphHash": 1, "symbolId": 1 },
{ name: "func_nodes_by_graph_symbol", unique: true, background: true }
);
db.func_nodes.createIndex(
{ "purl": 1, "symbolDigest": 1 },
{ name: "func_nodes_by_purl_symboldigest", background: true, sparse: true }
);
db.func_nodes.createIndex(
{ "codeId": 1 },
{ name: "func_nodes_by_code_id", background: true, sparse: true }
);
print(`- call_edges`);
db.call_edges.createIndex(
{ "graphHash": 1, "sourceId": 1, "targetId": 1, "type": 1 },
{ name: "call_edges_by_graph_edge", unique: true, background: true }
);
db.call_edges.createIndex(
{ "graphHash": 1, "sourceId": 1 },
{ name: "call_edges_by_graph_source", background: true }
);
db.call_edges.createIndex(
{ "graphHash": 1, "targetId": 1 },
{ name: "call_edges_by_graph_target", background: true }
);
db.call_edges.createIndex(
{ "purl": 1, "symbolDigest": 1 },
{ name: "call_edges_by_purl_symboldigest", background: true, sparse: true }
);
print(`- cve_func_hits`);
db.cve_func_hits.createIndex(
{ "subjectKey": 1, "cveId": 1 },
{ name: "cve_func_hits_by_subject_cve", background: true }
);
db.cve_func_hits.createIndex(
{ "cveId": 1, "purl": 1, "symbolDigest": 1 },
{ name: "cve_func_hits_by_cve_purl_symboldigest", background: true, sparse: true }
);
db.cve_func_hits.createIndex(
{ "graphHash": 1 },
{ name: "cve_func_hits_by_graph", background: true, sparse: true }
);
print("\nReachability store indexes created successfully.");
print("Run db.func_nodes.getIndexes(), db.call_edges.getIndexes(), db.cve_func_hits.getIndexes() to verify.");

125
devops/database/mongo/taskrunner/20251106-task-runner-baseline.mongosh Normal file
View File

@@ -0,0 +1,125 @@
// Task Runner baseline collections and indexes
// Mirrors docs/modules/taskrunner/migrations/pack-run-collections.md (last updated 2025-11-06)
function ensureCollection(name, validator) {
const existing = db.getCollectionNames();
if (!existing.includes(name)) {
db.createCollection(name, { validator, validationLevel: "moderate" });
} else if (validator) {
db.runCommand({ collMod: name, validator, validationLevel: "moderate" });
}
}
const runValidator = {
$jsonSchema: {
bsonType: "object",
required: ["planHash", "plan", "failurePolicy", "requestedAt", "createdAt", "updatedAt", "steps"],
properties: {
_id: { bsonType: "string" },
planHash: { bsonType: "string" },
plan: { bsonType: "object" },
failurePolicy: { bsonType: "object" },
requestedAt: { bsonType: "date" },
createdAt: { bsonType: "date" },
updatedAt: { bsonType: "date" },
steps: {
bsonType: "array",
items: {
bsonType: "object",
required: ["stepId", "status", "attempts"],
properties: {
stepId: { bsonType: "string" },
status: { bsonType: "string" },
attempts: { bsonType: "int" },
kind: { bsonType: "string" },
enabled: { bsonType: "bool" },
continueOnError: { bsonType: "bool" },
maxParallel: { bsonType: ["int", "null"] },
approvalId: { bsonType: ["string", "null"] },
gateMessage: { bsonType: ["string", "null"] },
lastTransitionAt: { bsonType: ["date", "null"] },
nextAttemptAt: { bsonType: ["date", "null"] },
statusReason: { bsonType: ["string", "null"] }
}
}
},
tenantId: { bsonType: ["string", "null"] }
}
}
};
const logValidator = {
$jsonSchema: {
bsonType: "object",
required: ["runId", "sequence", "timestamp", "level", "eventType", "message"],
properties: {
runId: { bsonType: "string" },
sequence: { bsonType: "long" },
timestamp: { bsonType: "date" },
level: { bsonType: "string" },
eventType: { bsonType: "string" },
message: { bsonType: "string" },
stepId: { bsonType: ["string", "null"] },
metadata: { bsonType: ["object", "null"] }
}
}
};
const artifactsValidator = {
$jsonSchema: {
bsonType: "object",
required: ["runId", "name", "type", "status", "capturedAt"],
properties: {
runId: { bsonType: "string" },
name: { bsonType: "string" },
type: { bsonType: "string" },
status: { bsonType: "string" },
capturedAt: { bsonType: "date" },
sourcePath: { bsonType: ["string", "null"] },
storedPath: { bsonType: ["string", "null"] },
notes: { bsonType: ["string", "null"] },
expression: { bsonType: ["object", "null"] }
}
}
};
const approvalsValidator = {
$jsonSchema: {
bsonType: "object",
required: ["runId", "approvalId", "requestedAt", "status"],
properties: {
runId: { bsonType: "string" },
approvalId: { bsonType: "string" },
requiredGrants: { bsonType: "array", items: { bsonType: "string" } },
stepIds: { bsonType: "array", items: { bsonType: "string" } },
messages: { bsonType: "array", items: { bsonType: "string" } },
reasonTemplate: { bsonType: ["string", "null"] },
requestedAt: { bsonType: "date" },
status: { bsonType: "string" },
actorId: { bsonType: ["string", "null"] },
completedAt: { bsonType: ["date", "null"] },
summary: { bsonType: ["string", "null"] }
}
}
};
ensureCollection("pack_runs", runValidator);
ensureCollection("pack_run_logs", logValidator);
ensureCollection("pack_artifacts", artifactsValidator);
ensureCollection("pack_run_approvals", approvalsValidator);
// Indexes for pack_runs
db.pack_runs.createIndex({ updatedAt: -1 }, { name: "pack_runs_updatedAt_desc" });
db.pack_runs.createIndex({ tenantId: 1, updatedAt: -1 }, { name: "pack_runs_tenant_updatedAt_desc", sparse: true });
// Indexes for pack_run_logs
db.pack_run_logs.createIndex({ runId: 1, sequence: 1 }, { unique: true, name: "pack_run_logs_run_sequence" });
db.pack_run_logs.createIndex({ runId: 1, timestamp: 1 }, { name: "pack_run_logs_run_timestamp" });
// Indexes for pack_artifacts
db.pack_artifacts.createIndex({ runId: 1, name: 1 }, { unique: true, name: "pack_artifacts_run_name" });
db.pack_artifacts.createIndex({ runId: 1 }, { name: "pack_artifacts_run" });
// Indexes for pack_run_approvals
db.pack_run_approvals.createIndex({ runId: 1, approvalId: 1 }, { unique: true, name: "pack_run_approvals_run_approval" });
db.pack_run_approvals.createIndex({ runId: 1, status: 1 }, { name: "pack_run_approvals_run_status" });

561
devops/database/postgres-partitioning/001_partition_infrastructure.sql Normal file
View File

@@ -0,0 +1,561 @@
-- Partitioning Infrastructure Migration 001: Foundation
-- Sprint: SPRINT_3422_0001_0001 - Time-Based Partitioning
-- Category: C (infrastructure setup, requires planned maintenance)
--
-- Purpose: Create partition management infrastructure including:
-- - Helper functions for partition creation and maintenance
-- - Utility functions for BRIN index optimization
-- - Partition maintenance scheduling support
--
-- This migration creates the foundation; table conversion is done in separate migrations.
BEGIN;
-- ============================================================================
-- Step 1: Create partition management schema
-- ============================================================================
CREATE SCHEMA IF NOT EXISTS partition_mgmt;
COMMENT ON SCHEMA partition_mgmt IS
'Partition management utilities for time-series tables';
-- ============================================================================
-- Step 2: Managed table registration
-- ============================================================================
CREATE TABLE IF NOT EXISTS partition_mgmt.managed_tables (
schema_name TEXT NOT NULL,
table_name TEXT NOT NULL,
partition_key TEXT NOT NULL,
partition_type TEXT NOT NULL,
retention_months INT NOT NULL DEFAULT 0,
months_ahead INT NOT NULL DEFAULT 3,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (schema_name, table_name)
);
COMMENT ON TABLE partition_mgmt.managed_tables IS
'Tracks partitioned tables with retention and creation settings';
-- ============================================================================
-- Step 3: Partition creation function
-- ============================================================================
-- Creates a new partition for a given table and date range
CREATE OR REPLACE FUNCTION partition_mgmt.create_partition(
p_schema_name TEXT,
p_table_name TEXT,
p_partition_column TEXT,
p_start_date DATE,
p_end_date DATE,
p_partition_suffix TEXT DEFAULT NULL
)
RETURNS TEXT
LANGUAGE plpgsql
AS $$
DECLARE
v_partition_name TEXT;
v_parent_table TEXT;
v_sql TEXT;
BEGIN
v_parent_table := format('%I.%I', p_schema_name, p_table_name);
-- Generate partition name: tablename_YYYY_MM or tablename_YYYY_Q#
IF p_partition_suffix IS NOT NULL THEN
v_partition_name := format('%s_%s', p_table_name, p_partition_suffix);
ELSE
v_partition_name := format('%s_%s', p_table_name, to_char(p_start_date, 'YYYY_MM'));
END IF;
-- Check if partition already exists
IF EXISTS (
SELECT 1 FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE n.nspname = p_schema_name AND c.relname = v_partition_name
) THEN
RAISE NOTICE 'Partition % already exists, skipping', v_partition_name;
RETURN v_partition_name;
END IF;
-- Create partition
v_sql := format(
'CREATE TABLE %I.%I PARTITION OF %s FOR VALUES FROM (%L) TO (%L)',
p_schema_name,
v_partition_name,
v_parent_table,
p_start_date,
p_end_date
);
EXECUTE v_sql;
RAISE NOTICE 'Created partition %.%', p_schema_name, v_partition_name;
RETURN v_partition_name;
END;
$$;
-- ============================================================================
-- Step 4: Monthly partition creation helper
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.create_monthly_partitions(
p_schema_name TEXT,
p_table_name TEXT,
p_partition_column TEXT,
p_start_month DATE,
p_months_ahead INT DEFAULT 3
)
RETURNS SETOF TEXT
LANGUAGE plpgsql
AS $$
DECLARE
v_current_month DATE;
v_end_month DATE;
v_partition_name TEXT;
BEGIN
v_current_month := date_trunc('month', p_start_month)::DATE;
v_end_month := date_trunc('month', NOW() + (p_months_ahead || ' months')::INTERVAL)::DATE;
WHILE v_current_month <= v_end_month LOOP
v_partition_name := partition_mgmt.create_partition(
p_schema_name,
p_table_name,
p_partition_column,
v_current_month,
(v_current_month + INTERVAL '1 month')::DATE
);
RETURN NEXT v_partition_name;
v_current_month := (v_current_month + INTERVAL '1 month')::DATE;
END LOOP;
END;
$$;
-- ============================================================================
-- Step 5: Quarterly partition creation helper
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.create_quarterly_partitions(
p_schema_name TEXT,
p_table_name TEXT,
p_partition_column TEXT,
p_start_quarter DATE,
p_quarters_ahead INT DEFAULT 2
)
RETURNS SETOF TEXT
LANGUAGE plpgsql
AS $$
DECLARE
v_current_quarter DATE;
v_end_quarter DATE;
v_partition_name TEXT;
v_suffix TEXT;
BEGIN
v_current_quarter := date_trunc('quarter', p_start_quarter)::DATE;
v_end_quarter := date_trunc('quarter', NOW() + (p_quarters_ahead * 3 || ' months')::INTERVAL)::DATE;
WHILE v_current_quarter <= v_end_quarter LOOP
-- Generate suffix like 2025_Q1, 2025_Q2, etc.
v_suffix := to_char(v_current_quarter, 'YYYY') || '_Q' ||
EXTRACT(QUARTER FROM v_current_quarter)::TEXT;
v_partition_name := partition_mgmt.create_partition(
p_schema_name,
p_table_name,
p_partition_column,
v_current_quarter,
(v_current_quarter + INTERVAL '3 months')::DATE,
v_suffix
);
RETURN NEXT v_partition_name;
v_current_quarter := (v_current_quarter + INTERVAL '3 months')::DATE;
END LOOP;
END;
$$;
-- ============================================================================
-- Step 6: Ensure future partitions exist
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.ensure_future_partitions(
p_schema_name TEXT,
p_table_name TEXT,
p_months_ahead INT
)
RETURNS INT
LANGUAGE plpgsql
AS $$
DECLARE
v_partition_key TEXT;
v_partition_type TEXT;
v_months_ahead INT;
v_created INT := 0;
v_current DATE;
v_end DATE;
v_suffix TEXT;
v_partition_name TEXT;
BEGIN
SELECT partition_key, partition_type, months_ahead
INTO v_partition_key, v_partition_type, v_months_ahead
FROM partition_mgmt.managed_tables
WHERE schema_name = p_schema_name
AND table_name = p_table_name;
IF v_partition_key IS NULL THEN
RETURN 0;
END IF;
IF p_months_ahead IS NOT NULL AND p_months_ahead > 0 THEN
v_months_ahead := p_months_ahead;
END IF;
IF v_months_ahead IS NULL OR v_months_ahead <= 0 THEN
RETURN 0;
END IF;
v_partition_type := lower(coalesce(v_partition_type, 'monthly'));
IF v_partition_type = 'monthly' THEN
v_current := date_trunc('month', NOW())::DATE;
v_end := date_trunc('month', NOW() + (v_months_ahead || ' months')::INTERVAL)::DATE;
WHILE v_current <= v_end LOOP
v_partition_name := format('%s_%s', p_table_name, to_char(v_current, 'YYYY_MM'));
IF NOT EXISTS (
SELECT 1 FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE n.nspname = p_schema_name AND c.relname = v_partition_name
) THEN
PERFORM partition_mgmt.create_partition(
p_schema_name,
p_table_name,
v_partition_key,
v_current,
(v_current + INTERVAL '1 month')::DATE
);
v_created := v_created + 1;
END IF;
v_current := (v_current + INTERVAL '1 month')::DATE;
END LOOP;
ELSIF v_partition_type = 'quarterly' THEN
v_current := date_trunc('quarter', NOW())::DATE;
v_end := date_trunc('quarter', NOW() + (v_months_ahead || ' months')::INTERVAL)::DATE;
WHILE v_current <= v_end LOOP
v_suffix := to_char(v_current, 'YYYY') || '_Q' ||
EXTRACT(QUARTER FROM v_current)::TEXT;
v_partition_name := format('%s_%s', p_table_name, v_suffix);
IF NOT EXISTS (
SELECT 1 FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE n.nspname = p_schema_name AND c.relname = v_partition_name
) THEN
PERFORM partition_mgmt.create_partition(
p_schema_name,
p_table_name,
v_partition_key,
v_current,
(v_current + INTERVAL '3 months')::DATE,
v_suffix
);
v_created := v_created + 1;
END IF;
v_current := (v_current + INTERVAL '3 months')::DATE;
END LOOP;
END IF;
RETURN v_created;
END;
$$;
-- ============================================================================
-- Step 7: Retention enforcement function
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.enforce_retention(
p_schema_name TEXT,
p_table_name TEXT,
p_retention_months INT
)
RETURNS INT
LANGUAGE plpgsql
AS $$
DECLARE
v_retention_months INT;
v_cutoff_date DATE;
v_partition RECORD;
v_dropped INT := 0;
BEGIN
SELECT retention_months
INTO v_retention_months
FROM partition_mgmt.managed_tables
WHERE schema_name = p_schema_name
AND table_name = p_table_name;
IF p_retention_months IS NOT NULL AND p_retention_months > 0 THEN
v_retention_months := p_retention_months;
END IF;
IF v_retention_months IS NULL OR v_retention_months <= 0 THEN
RETURN 0;
END IF;
v_cutoff_date := (NOW() - (v_retention_months || ' months')::INTERVAL)::DATE;
FOR v_partition IN
SELECT partition_name, partition_end
FROM partition_mgmt.partition_stats
WHERE schema_name = p_schema_name
AND table_name = p_table_name
LOOP
IF v_partition.partition_end IS NOT NULL AND v_partition.partition_end < v_cutoff_date THEN
EXECUTE format('DROP TABLE IF EXISTS %I.%I', p_schema_name, v_partition.partition_name);
v_dropped := v_dropped + 1;
END IF;
END LOOP;
RETURN v_dropped;
END;
$$;
-- ============================================================================
-- Step 8: Partition detach and archive function
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.detach_partition(
p_schema_name TEXT,
p_table_name TEXT,
p_partition_name TEXT,
p_archive_schema TEXT DEFAULT 'archive'
)
RETURNS BOOLEAN
LANGUAGE plpgsql
AS $$
DECLARE
v_parent_table TEXT;
v_partition_full TEXT;
v_archive_table TEXT;
BEGIN
v_parent_table := format('%I.%I', p_schema_name, p_table_name);
v_partition_full := format('%I.%I', p_schema_name, p_partition_name);
v_archive_table := format('%I.%I', p_archive_schema, p_partition_name);
-- Create archive schema if not exists
EXECUTE format('CREATE SCHEMA IF NOT EXISTS %I', p_archive_schema);
-- Detach partition
EXECUTE format(
'ALTER TABLE %s DETACH PARTITION %s',
v_parent_table,
v_partition_full
);
-- Move to archive schema
EXECUTE format(
'ALTER TABLE %s SET SCHEMA %I',
v_partition_full,
p_archive_schema
);
RAISE NOTICE 'Detached and archived partition % to %', p_partition_name, v_archive_table;
RETURN TRUE;
EXCEPTION
WHEN OTHERS THEN
RAISE WARNING 'Failed to detach partition %: %', p_partition_name, SQLERRM;
RETURN FALSE;
END;
$$;
-- ============================================================================
-- Step 9: Partition retention cleanup function
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.cleanup_old_partitions(
p_schema_name TEXT,
p_table_name TEXT,
p_retention_months INT,
p_archive_schema TEXT DEFAULT 'archive',
p_dry_run BOOLEAN DEFAULT TRUE
)
RETURNS TABLE(partition_name TEXT, action TEXT)
LANGUAGE plpgsql
AS $$
DECLARE
v_cutoff_date DATE;
v_partition RECORD;
v_partition_end DATE;
BEGIN
v_cutoff_date := (NOW() - (p_retention_months || ' months')::INTERVAL)::DATE;
FOR v_partition IN
SELECT c.relname as name,
pg_get_expr(c.relpartbound, c.oid) as bound_expr
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE n.nspname = p_schema_name
AND parent.relname = p_table_name
AND c.relkind = 'r'
LOOP
-- Parse the partition bound to get end date
-- Format: FOR VALUES FROM ('2024-01-01') TO ('2024-02-01')
v_partition_end := (regexp_match(v_partition.bound_expr,
'TO \(''([^'']+)''\)'))[1]::DATE;
IF v_partition_end IS NOT NULL AND v_partition_end < v_cutoff_date THEN
partition_name := v_partition.name;
IF p_dry_run THEN
action := 'WOULD_ARCHIVE';
ELSE
IF partition_mgmt.detach_partition(
p_schema_name, p_table_name, v_partition.name, p_archive_schema
) THEN
action := 'ARCHIVED';
ELSE
action := 'FAILED';
END IF;
END IF;
RETURN NEXT;
END IF;
END LOOP;
END;
$$;
-- ============================================================================
-- Step 10: Partition statistics view
-- ============================================================================
CREATE OR REPLACE VIEW partition_mgmt.partition_stats AS
SELECT
n.nspname AS schema_name,
parent.relname AS table_name,
c.relname AS partition_name,
pg_get_expr(c.relpartbound, c.oid) AS partition_range,
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \(''([^'']+)''\)'))[1]::DATE AS partition_start,
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \(''([^'']+)''\)'))[1]::DATE AS partition_end,
pg_size_pretty(pg_relation_size(c.oid)) AS size,
pg_relation_size(c.oid) AS size_bytes,
COALESCE(s.n_live_tup, 0) AS estimated_rows,
s.last_vacuum,
s.last_autovacuum,
s.last_analyze,
s.last_autoanalyze
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
LEFT JOIN pg_stat_user_tables s ON c.oid = s.relid
WHERE c.relkind = 'r'
AND parent.relkind = 'p'
ORDER BY n.nspname, parent.relname, c.relname;
COMMENT ON VIEW partition_mgmt.partition_stats IS
'Statistics for all partitioned tables in the database';
-- ============================================================================
-- Step 11: BRIN index optimization helper
-- ============================================================================
CREATE OR REPLACE FUNCTION partition_mgmt.create_brin_index_if_not_exists(
p_schema_name TEXT,
p_table_name TEXT,
p_column_name TEXT,
p_pages_per_range INT DEFAULT 128
)
RETURNS BOOLEAN
LANGUAGE plpgsql
AS $$
DECLARE
v_index_name TEXT;
v_sql TEXT;
BEGIN
v_index_name := format('brin_%s_%s', p_table_name, p_column_name);
-- Check if index exists
IF EXISTS (
SELECT 1 FROM pg_indexes
WHERE schemaname = p_schema_name AND indexname = v_index_name
) THEN
RAISE NOTICE 'BRIN index % already exists', v_index_name;
RETURN FALSE;
END IF;
v_sql := format(
'CREATE INDEX %I ON %I.%I USING brin (%I) WITH (pages_per_range = %s)',
v_index_name,
p_schema_name,
p_table_name,
p_column_name,
p_pages_per_range
);
EXECUTE v_sql;
RAISE NOTICE 'Created BRIN index % on %.%(%)',
v_index_name, p_schema_name, p_table_name, p_column_name;
RETURN TRUE;
END;
$$;
-- ============================================================================
-- Step 12: Maintenance job tracking table
-- ============================================================================
CREATE TABLE IF NOT EXISTS partition_mgmt.maintenance_log (
id BIGSERIAL PRIMARY KEY,
operation TEXT NOT NULL,
schema_name TEXT NOT NULL,
table_name TEXT NOT NULL,
partition_name TEXT,
status TEXT NOT NULL DEFAULT 'started',
details JSONB NOT NULL DEFAULT '{}',
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
error_message TEXT
);
CREATE INDEX idx_maintenance_log_table ON partition_mgmt.maintenance_log(schema_name, table_name);
CREATE INDEX idx_maintenance_log_status ON partition_mgmt.maintenance_log(status, started_at);
-- ============================================================================
-- Step 13: Archive schema for detached partitions
-- ============================================================================
CREATE SCHEMA IF NOT EXISTS archive;
COMMENT ON SCHEMA archive IS
'Storage for detached/archived partitions awaiting deletion or offload';
COMMIT;
-- ============================================================================
-- Usage Examples (commented out)
-- ============================================================================
/*
-- Create monthly partitions for audit table, 3 months ahead
SELECT partition_mgmt.create_monthly_partitions(
'scheduler', 'audit', 'created_at', '2024-01-01'::DATE, 3
);
-- Preview old partitions that would be archived (dry run)
SELECT * FROM partition_mgmt.cleanup_old_partitions(
'scheduler', 'audit', 12, 'archive', TRUE
);
-- Actually archive old partitions
SELECT * FROM partition_mgmt.cleanup_old_partitions(
'scheduler', 'audit', 12, 'archive', FALSE
);
-- View partition statistics
SELECT * FROM partition_mgmt.partition_stats
WHERE schema_name = 'scheduler'
ORDER BY table_name, partition_name;
*/

143
devops/database/postgres-partitioning/002_calibration_schema.sql Normal file
View File

@@ -0,0 +1,143 @@
-- Migration: Trust Vector Calibration Schema
-- Sprint: 7100.0002.0002
-- Description: Creates schema and tables for trust vector calibration system
-- Create calibration schema
CREATE SCHEMA IF NOT EXISTS excititor_calibration;
-- Calibration manifests table
-- Stores signed manifests for each calibration epoch
CREATE TABLE IF NOT EXISTS excititor_calibration.calibration_manifests (
manifest_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL,
epoch_number INTEGER NOT NULL,
epoch_start_utc TIMESTAMP NOT NULL,
epoch_end_utc TIMESTAMP NOT NULL,
sample_count INTEGER NOT NULL,
learning_rate DOUBLE PRECISION NOT NULL,
policy_hash TEXT,
lattice_version TEXT NOT NULL,
manifest_json JSONB NOT NULL,
signature_envelope JSONB,
created_at_utc TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
created_by TEXT NOT NULL,
CONSTRAINT uq_calibration_manifest_tenant_epoch UNIQUE (tenant_id, epoch_number)
);
CREATE INDEX idx_calibration_manifests_tenant
ON excititor_calibration.calibration_manifests(tenant_id);
CREATE INDEX idx_calibration_manifests_created
ON excititor_calibration.calibration_manifests(created_at_utc DESC);
-- Trust vector adjustments table
-- Records each provider's trust vector changes per epoch
CREATE TABLE IF NOT EXISTS excititor_calibration.trust_vector_adjustments (
adjustment_id BIGSERIAL PRIMARY KEY,
manifest_id TEXT NOT NULL REFERENCES excititor_calibration.calibration_manifests(manifest_id),
source_id TEXT NOT NULL,
old_provenance DOUBLE PRECISION NOT NULL,
old_coverage DOUBLE PRECISION NOT NULL,
old_replayability DOUBLE PRECISION NOT NULL,
new_provenance DOUBLE PRECISION NOT NULL,
new_coverage DOUBLE PRECISION NOT NULL,
new_replayability DOUBLE PRECISION NOT NULL,
adjustment_magnitude DOUBLE PRECISION NOT NULL,
confidence_in_adjustment DOUBLE PRECISION NOT NULL,
sample_count_for_source INTEGER NOT NULL,
created_at_utc TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
CONSTRAINT chk_old_provenance_range CHECK (old_provenance >= 0 AND old_provenance <= 1),
CONSTRAINT chk_old_coverage_range CHECK (old_coverage >= 0 AND old_coverage <= 1),
CONSTRAINT chk_old_replayability_range CHECK (old_replayability >= 0 AND old_replayability <= 1),
CONSTRAINT chk_new_provenance_range CHECK (new_provenance >= 0 AND new_provenance <= 1),
CONSTRAINT chk_new_coverage_range CHECK (new_coverage >= 0 AND new_coverage <= 1),
CONSTRAINT chk_new_replayability_range CHECK (new_replayability >= 0 AND new_replayability <= 1),
CONSTRAINT chk_confidence_range CHECK (confidence_in_adjustment >= 0 AND confidence_in_adjustment <= 1)
);
CREATE INDEX idx_trust_adjustments_manifest
ON excititor_calibration.trust_vector_adjustments(manifest_id);
CREATE INDEX idx_trust_adjustments_source
ON excititor_calibration.trust_vector_adjustments(source_id);
-- Calibration feedback samples table
-- Stores empirical evidence used for calibration
CREATE TABLE IF NOT EXISTS excititor_calibration.calibration_samples (
sample_id BIGSERIAL PRIMARY KEY,
tenant_id TEXT NOT NULL,
source_id TEXT NOT NULL,
cve_id TEXT NOT NULL,
purl TEXT NOT NULL,
expected_status TEXT NOT NULL,
actual_status TEXT NOT NULL,
verdict_confidence DOUBLE PRECISION NOT NULL,
is_match BOOLEAN NOT NULL,
feedback_source TEXT NOT NULL, -- 'reachability', 'customer_feedback', 'integration_tests'
feedback_weight DOUBLE PRECISION NOT NULL DEFAULT 1.0,
scan_id TEXT,
collected_at_utc TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
processed BOOLEAN NOT NULL DEFAULT FALSE,
processed_in_manifest_id TEXT REFERENCES excititor_calibration.calibration_manifests(manifest_id),
CONSTRAINT chk_verdict_confidence_range CHECK (verdict_confidence >= 0 AND verdict_confidence <= 1),
CONSTRAINT chk_feedback_weight_range CHECK (feedback_weight >= 0 AND feedback_weight <= 1)
);
CREATE INDEX idx_calibration_samples_tenant
ON excititor_calibration.calibration_samples(tenant_id);
CREATE INDEX idx_calibration_samples_source
ON excititor_calibration.calibration_samples(source_id);
CREATE INDEX idx_calibration_samples_collected
ON excititor_calibration.calibration_samples(collected_at_utc DESC);
CREATE INDEX idx_calibration_samples_processed
ON excititor_calibration.calibration_samples(processed) WHERE NOT processed;
-- Calibration metrics table
-- Tracks performance metrics per source/severity/status
CREATE TABLE IF NOT EXISTS excititor_calibration.calibration_metrics (
metric_id BIGSERIAL PRIMARY KEY,
manifest_id TEXT NOT NULL REFERENCES excititor_calibration.calibration_manifests(manifest_id),
source_id TEXT,
severity TEXT,
status TEXT,
precision DOUBLE PRECISION NOT NULL,
recall DOUBLE PRECISION NOT NULL,
f1_score DOUBLE PRECISION NOT NULL,
false_positive_rate DOUBLE PRECISION NOT NULL,
false_negative_rate DOUBLE PRECISION NOT NULL,
sample_count INTEGER NOT NULL,
created_at_utc TIMESTAMP NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
CONSTRAINT chk_precision_range CHECK (precision >= 0 AND precision <= 1),
CONSTRAINT chk_recall_range CHECK (recall >= 0 AND recall <= 1),
CONSTRAINT chk_f1_range CHECK (f1_score >= 0 AND f1_score <= 1),
CONSTRAINT chk_fpr_range CHECK (false_positive_rate >= 0 AND false_positive_rate <= 1),
CONSTRAINT chk_fnr_range CHECK (false_negative_rate >= 0 AND false_negative_rate <= 1)
);
CREATE INDEX idx_calibration_metrics_manifest
ON excititor_calibration.calibration_metrics(manifest_id);
CREATE INDEX idx_calibration_metrics_source
ON excititor_calibration.calibration_metrics(source_id) WHERE source_id IS NOT NULL;
-- Grant permissions to excititor service role
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'excititor_service') THEN
GRANT USAGE ON SCHEMA excititor_calibration TO excititor_service;
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA excititor_calibration TO excititor_service;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA excititor_calibration TO excititor_service;
ALTER DEFAULT PRIVILEGES IN SCHEMA excititor_calibration
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO excititor_service;
ALTER DEFAULT PRIVILEGES IN SCHEMA excititor_calibration
GRANT USAGE, SELECT ON SEQUENCES TO excititor_service;
END IF;
END $$;
-- Comments for documentation
COMMENT ON SCHEMA excititor_calibration IS 'Trust vector calibration data for VEX source scoring';
COMMENT ON TABLE excititor_calibration.calibration_manifests IS 'Signed calibration epoch results';
COMMENT ON TABLE excititor_calibration.trust_vector_adjustments IS 'Per-source trust vector changes per epoch';
COMMENT ON TABLE excititor_calibration.calibration_samples IS 'Empirical feedback samples for calibration';
COMMENT ON TABLE excititor_calibration.calibration_metrics IS 'Performance metrics per calibration epoch';

97
devops/database/postgres-partitioning/provcache/create_provcache_schema.sql Normal file
View File

@@ -0,0 +1,97 @@
-- Provcache schema migration
-- Run as: psql -d stellaops -f create_provcache_schema.sql
-- Create schema
CREATE SCHEMA IF NOT EXISTS provcache;
-- Main cache items table
CREATE TABLE IF NOT EXISTS provcache.provcache_items (
verikey TEXT PRIMARY KEY,
digest_version TEXT NOT NULL DEFAULT 'v1',
verdict_hash TEXT NOT NULL,
proof_root TEXT NOT NULL,
replay_seed JSONB NOT NULL,
policy_hash TEXT NOT NULL,
signer_set_hash TEXT NOT NULL,
feed_epoch TEXT NOT NULL,
trust_score INTEGER NOT NULL CHECK (trust_score >= 0 AND trust_score <= 100),
hit_count BIGINT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
last_accessed_at TIMESTAMPTZ,
-- Constraint: expires_at must be after created_at
CONSTRAINT provcache_items_expires_check CHECK (expires_at > created_at)
);
-- Indexes for invalidation queries
CREATE INDEX IF NOT EXISTS idx_provcache_policy_hash
ON provcache.provcache_items(policy_hash);
CREATE INDEX IF NOT EXISTS idx_provcache_signer_set_hash
ON provcache.provcache_items(signer_set_hash);
CREATE INDEX IF NOT EXISTS idx_provcache_feed_epoch
ON provcache.provcache_items(feed_epoch);
CREATE INDEX IF NOT EXISTS idx_provcache_expires_at
ON provcache.provcache_items(expires_at);
CREATE INDEX IF NOT EXISTS idx_provcache_created_at
ON provcache.provcache_items(created_at);
-- Evidence chunks table for large evidence storage
CREATE TABLE IF NOT EXISTS provcache.prov_evidence_chunks (
chunk_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
proof_root TEXT NOT NULL,
chunk_index INTEGER NOT NULL,
chunk_hash TEXT NOT NULL,
blob BYTEA NOT NULL,
blob_size INTEGER NOT NULL,
content_type TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT prov_evidence_chunks_unique_index
UNIQUE (proof_root, chunk_index)
);
CREATE INDEX IF NOT EXISTS idx_prov_chunks_proof_root
ON provcache.prov_evidence_chunks(proof_root);
-- Revocation audit log
CREATE TABLE IF NOT EXISTS provcache.prov_revocations (
revocation_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
revocation_type TEXT NOT NULL,
target_hash TEXT NOT NULL,
reason TEXT,
actor TEXT,
entries_affected BIGINT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_prov_revocations_created_at
ON provcache.prov_revocations(created_at);
CREATE INDEX IF NOT EXISTS idx_prov_revocations_target_hash
ON provcache.prov_revocations(target_hash);
-- Function to update updated_at timestamp
CREATE OR REPLACE FUNCTION provcache.update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ language 'plpgsql';
-- Trigger for auto-updating updated_at
DROP TRIGGER IF EXISTS update_provcache_items_updated_at ON provcache.provcache_items;
CREATE TRIGGER update_provcache_items_updated_at
BEFORE UPDATE ON provcache.provcache_items
FOR EACH ROW
EXECUTE FUNCTION provcache.update_updated_at_column();
-- Grant permissions (adjust role as needed)
-- GRANT USAGE ON SCHEMA provcache TO stellaops_app;
-- GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA provcache TO stellaops_app;
-- GRANT USAGE ON ALL SEQUENCES IN SCHEMA provcache TO stellaops_app;
COMMENT ON TABLE provcache.provcache_items IS 'Provenance cache entries for cached security decisions';
COMMENT ON TABLE provcache.prov_evidence_chunks IS 'Chunked evidence storage for large SBOMs and attestations';
COMMENT ON TABLE provcache.prov_revocations IS 'Audit log of cache invalidation events';

159
devops/database/postgres-validation/001_validate_rls.sql Normal file
View File

@@ -0,0 +1,159 @@
-- RLS Validation Script
-- Sprint: SPRINT_3421_0001_0001 - RLS Expansion
--
-- Purpose: Verify that RLS is properly configured on all tenant-scoped tables
-- Run this script after deploying RLS migrations to validate configuration
-- ============================================================================
-- Part 1: List all tables with RLS status
-- ============================================================================
\echo '=== RLS Status for All Schemas ==='
SELECT
schemaname AS schema,
tablename AS table_name,
rowsecurity AS rls_enabled,
forcerowsecurity AS rls_forced,
CASE
WHEN rowsecurity AND forcerowsecurity THEN 'OK'
WHEN rowsecurity AND NOT forcerowsecurity THEN 'WARN: Not forced'
ELSE 'MISSING'
END AS status
FROM pg_tables
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
ORDER BY schemaname, tablename;
-- ============================================================================
-- Part 2: List all RLS policies
-- ============================================================================
\echo ''
\echo '=== RLS Policies ==='
SELECT
schemaname AS schema,
tablename AS table_name,
policyname AS policy_name,
permissive,
roles,
cmd AS applies_to,
qual IS NOT NULL AS has_using,
with_check IS NOT NULL AS has_check
FROM pg_policies
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
ORDER BY schemaname, tablename, policyname;
-- ============================================================================
-- Part 3: Tables missing RLS that should have it (have tenant_id column)
-- ============================================================================
\echo ''
\echo '=== Tables with tenant_id but NO RLS ==='
SELECT
c.table_schema AS schema,
c.table_name AS table_name,
'MISSING RLS' AS issue
FROM information_schema.columns c
JOIN pg_tables t ON c.table_schema = t.schemaname AND c.table_name = t.tablename
WHERE c.column_name IN ('tenant_id', 'tenant')
AND c.table_schema IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
AND NOT t.rowsecurity
ORDER BY c.table_schema, c.table_name;
-- ============================================================================
-- Part 4: Verify helper functions exist
-- ============================================================================
\echo ''
\echo '=== RLS Helper Functions ==='
SELECT
n.nspname AS schema,
p.proname AS function_name,
CASE
WHEN p.prosecdef THEN 'SECURITY DEFINER'
ELSE 'SECURITY INVOKER'
END AS security,
CASE
WHEN p.provolatile = 's' THEN 'STABLE'
WHEN p.provolatile = 'i' THEN 'IMMUTABLE'
ELSE 'VOLATILE'
END AS volatility
FROM pg_proc p
JOIN pg_namespace n ON p.pronamespace = n.oid
WHERE p.proname = 'require_current_tenant'
AND n.nspname LIKE '%_app'
ORDER BY n.nspname;
-- ============================================================================
-- Part 5: Test RLS enforcement (expect failure without tenant context)
-- ============================================================================
\echo ''
\echo '=== RLS Enforcement Test ==='
\echo 'Testing RLS on scheduler.runs (should fail without tenant context)...'
-- Reset tenant context
SELECT set_config('app.tenant_id', '', false);
DO $$
BEGIN
-- This should raise an exception if RLS is working
PERFORM * FROM scheduler.runs LIMIT 1;
RAISE NOTICE 'WARNING: Query succeeded without tenant context - RLS may not be working!';
EXCEPTION
WHEN OTHERS THEN
RAISE NOTICE 'OK: RLS blocked query without tenant context: %', SQLERRM;
END
$$;
-- ============================================================================
-- Part 6: Admin bypass role verification
-- ============================================================================
\echo ''
\echo '=== Admin Bypass Roles ==='
SELECT
rolname AS role_name,
rolbypassrls AS can_bypass_rls,
rolcanlogin AS can_login
FROM pg_roles
WHERE rolname LIKE '%_admin'
AND rolbypassrls = TRUE
ORDER BY rolname;
-- ============================================================================
-- Summary
-- ============================================================================
\echo ''
\echo '=== Summary ==='
SELECT
'Total Tables' AS metric,
COUNT(*)::TEXT AS value
FROM pg_tables
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
UNION ALL
SELECT
'Tables with RLS Enabled',
COUNT(*)::TEXT
FROM pg_tables
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
AND rowsecurity = TRUE
UNION ALL
SELECT
'Tables with RLS Forced',
COUNT(*)::TEXT
FROM pg_tables
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns')
AND forcerowsecurity = TRUE
UNION ALL
SELECT
'Active Policies',
COUNT(*)::TEXT
FROM pg_policies
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns');

238
devops/database/postgres-validation/002_validate_partitions.sql Normal file
View File

@@ -0,0 +1,238 @@
-- Partition Validation Script
-- Sprint: SPRINT_3422_0001_0001 - Time-Based Partitioning
--
-- Purpose: Verify that partitioned tables are properly configured and healthy
-- ============================================================================
-- Part 1: List all partitioned tables
-- ============================================================================
\echo '=== Partitioned Tables ==='
SELECT
n.nspname AS schema,
c.relname AS table_name,
CASE pt.partstrat
WHEN 'r' THEN 'RANGE'
WHEN 'l' THEN 'LIST'
WHEN 'h' THEN 'HASH'
END AS partition_strategy,
array_to_string(array_agg(a.attname ORDER BY k.col), ', ') AS partition_key
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_partitioned_table pt ON c.oid = pt.partrelid
JOIN LATERAL unnest(pt.partattrs) WITH ORDINALITY AS k(col, idx) ON true
LEFT JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.col
WHERE n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
GROUP BY n.nspname, c.relname, pt.partstrat
ORDER BY n.nspname, c.relname;
-- ============================================================================
-- Part 2: Partition inventory with sizes
-- ============================================================================
\echo ''
\echo '=== Partition Inventory ==='
SELECT
n.nspname AS schema,
parent.relname AS parent_table,
c.relname AS partition_name,
pg_get_expr(c.relpartbound, c.oid) AS bounds,
pg_size_pretty(pg_relation_size(c.oid)) AS size,
s.n_live_tup AS estimated_rows
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
LEFT JOIN pg_stat_user_tables s ON c.oid = s.relid
WHERE n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
AND c.relkind = 'r'
AND parent.relkind = 'p'
ORDER BY n.nspname, parent.relname, c.relname;
-- ============================================================================
-- Part 3: Check for missing future partitions
-- ============================================================================
\echo ''
\echo '=== Future Partition Coverage ==='
WITH partition_bounds AS (
SELECT
n.nspname AS schema_name,
parent.relname AS table_name,
c.relname AS partition_name,
-- Extract the TO date from partition bound
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \(''([^'']+)''\)'))[1]::DATE AS end_date
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE c.relkind = 'r'
AND parent.relkind = 'p'
AND c.relname NOT LIKE '%_default'
),
max_bounds AS (
SELECT
schema_name,
table_name,
MAX(end_date) AS max_partition_date
FROM partition_bounds
WHERE end_date IS NOT NULL
GROUP BY schema_name, table_name
)
SELECT
schema_name,
table_name,
max_partition_date,
(max_partition_date - CURRENT_DATE) AS days_ahead,
CASE
WHEN (max_partition_date - CURRENT_DATE) < 30 THEN 'CRITICAL: Create partitions!'
WHEN (max_partition_date - CURRENT_DATE) < 60 THEN 'WARNING: Running low'
ELSE 'OK'
END AS status
FROM max_bounds
ORDER BY days_ahead;
-- ============================================================================
-- Part 4: Check for orphaned data in default partitions
-- ============================================================================
\echo ''
\echo '=== Default Partition Data (should be empty) ==='
DO $$
DECLARE
v_schema TEXT;
v_table TEXT;
v_count BIGINT;
v_sql TEXT;
BEGIN
FOR v_schema, v_table IN
SELECT n.nspname, c.relname
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
WHERE c.relname LIKE '%_default'
AND n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
LOOP
v_sql := format('SELECT COUNT(*) FROM %I.%I', v_schema, v_table);
EXECUTE v_sql INTO v_count;
IF v_count > 0 THEN
RAISE NOTICE 'WARNING: %.% has % rows in default partition!',
v_schema, v_table, v_count;
ELSE
RAISE NOTICE 'OK: %.% is empty', v_schema, v_table;
END IF;
END LOOP;
END
$$;
-- ============================================================================
-- Part 5: Index health on partitions
-- ============================================================================
\echo ''
\echo '=== Partition Index Coverage ==='
SELECT
schemaname AS schema,
tablename AS table_name,
indexname AS index_name,
indexdef
FROM pg_indexes
WHERE schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
AND tablename LIKE '%_partitioned' OR tablename LIKE '%_202%'
ORDER BY schemaname, tablename, indexname;
-- ============================================================================
-- Part 6: BRIN index effectiveness check
-- ============================================================================
\echo ''
\echo '=== BRIN Index Statistics ==='
SELECT
schemaname AS schema,
tablename AS table_name,
indexrelname AS index_name,
idx_scan AS scans,
idx_tup_read AS tuples_read,
idx_tup_fetch AS tuples_fetched,
pg_size_pretty(pg_relation_size(indexrelid)) AS index_size
FROM pg_stat_user_indexes
WHERE indexrelname LIKE 'brin_%'
ORDER BY schemaname, tablename;
-- ============================================================================
-- Part 7: Partition maintenance recommendations
-- ============================================================================
\echo ''
\echo '=== Maintenance Recommendations ==='
WITH partition_ages AS (
SELECT
n.nspname AS schema_name,
parent.relname AS table_name,
c.relname AS partition_name,
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \(''([^'']+)''\)'))[1]::DATE AS start_date,
(regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \(''([^'']+)''\)'))[1]::DATE AS end_date
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE c.relkind = 'r'
AND parent.relkind = 'p'
AND c.relname NOT LIKE '%_default'
)
SELECT
schema_name,
table_name,
partition_name,
start_date,
end_date,
(CURRENT_DATE - end_date) AS days_old,
CASE
WHEN (CURRENT_DATE - end_date) > 365 THEN 'Consider archiving (>1 year old)'
WHEN (CURRENT_DATE - end_date) > 180 THEN 'Review retention policy (>6 months old)'
ELSE 'Current'
END AS recommendation
FROM partition_ages
WHERE start_date IS NOT NULL
ORDER BY schema_name, table_name, start_date;
-- ============================================================================
-- Summary
-- ============================================================================
\echo ''
\echo '=== Summary ==='
SELECT
'Partitioned Tables' AS metric,
COUNT(DISTINCT parent.relname)::TEXT AS value
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
AND parent.relkind = 'p'
UNION ALL
SELECT
'Total Partitions',
COUNT(*)::TEXT
FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid
JOIN pg_inherits i ON c.oid = i.inhrelid
JOIN pg_class parent ON i.inhparent = parent.oid
WHERE n.nspname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln')
AND parent.relkind = 'p'
UNION ALL
SELECT
'BRIN Indexes',
COUNT(*)::TEXT
FROM pg_indexes
WHERE indexname LIKE 'brin_%'
AND schemaname IN ('scheduler', 'notify', 'authority', 'vex', 'policy', 'unknowns', 'vuln');

66
devops/database/postgres/README.md Normal file
View File

@@ -0,0 +1,66 @@
# PostgreSQL 16 Cluster (staging / production)
This directory provisions StellaOps PostgreSQL clusters with **CloudNativePG (CNPG)**. It is pinned to Postgres 16.x, includes connection pooling (PgBouncer), Prometheus scraping, and S3-compatible backups. Everything is air-gap friendly: fetch the operator and images once, then render/apply manifests offline.
## Targets
- **Staging:** `stellaops-pg-stg` (2 instances, 200 Gi data, WAL 64 Gi, PgBouncer x2)
- **Production:** `stellaops-pg-prod` (3 instances, 500 Gi data, WAL 128 Gi, PgBouncer x3)
- **Namespace:** `platform-postgres`
## Prerequisites
- Kubernetes ≥ 1.27 with CSI storage classes `fast-ssd` (data) and `fast-wal` (WAL) available.
- CloudNativePG operator 1.23.x mirrored or downloaded to `artifacts/cloudnative-pg-1.23.0.yaml`.
- Images mirrored to your registry (example tags):
- `ghcr.io/cloudnative-pg/postgresql:16.4`
- `ghcr.io/cloudnative-pg/postgresql-operator:1.23.0`
- `ghcr.io/cloudnative-pg/pgbouncer:1.23.0`
- Secrets created from the templates under `ops/devops/postgres/secrets/` (superuser, app user, backup credentials).
## Render & Apply (deterministic)
```bash
# 1) Create namespace
kubectl apply -f ops/devops/postgres/namespace.yaml
# 2) Install operator (offline-friendly: use the pinned manifest you mirrored)
kubectl apply -f artifacts/cloudnative-pg-1.23.0.yaml
# 3) Create secrets (replace passwords/keys first)
kubectl apply -f ops/devops/postgres/secrets/example-superuser.yaml
kubectl apply -f ops/devops/postgres/secrets/example-app.yaml
kubectl apply -f ops/devops/postgres/secrets/example-backup-credentials.yaml
# 4) Apply the cluster and pooler for the target environment
kubectl apply -f ops/devops/postgres/cluster-staging.yaml
kubectl apply -f ops/devops/postgres/pooler-staging.yaml
# or
kubectl apply -f ops/devops/postgres/cluster-production.yaml
kubectl apply -f ops/devops/postgres/pooler-production.yaml
```
## Connection Endpoints
- RW service: `<cluster>-rw` (e.g., `stellaops-pg-stg-rw:5432`)
- RO service: `<cluster>-ro`
- PgBouncer pooler: `<pooler-name>` (e.g., `stellaops-pg-stg-pooler:6432`)
**Application connection string (matches library defaults):**
`Host=stellaops-pg-stg-pooler;Port=6432;Username=stellaops_app;Password=<app-password>;Database=stellaops;Pooling=true;Timeout=15;CommandTimeout=30;Ssl Mode=Require;`
## Monitoring & Backups
- `monitoring.enablePodMonitor: true` exposes PodMonitor for Prometheus Operator.
- Barman/S3 backups are enabled by default; set `backup.barmanObjectStore.destinationPath` per env and populate `stellaops-pg-backup` credentials.
- WAL compression is `gzip`; retention is operator-managed (configure via Barman bucket policies).
## Alignment with code defaults
- Session settings: UTC timezone, 30s `statement_timeout`, tenant context via `set_config('app.current_tenant', ...)`.
- Connection pooler uses **transaction** mode with a `server_reset_query` that clears session state, keeping RepositoryBase deterministic.
## Verification checklist
- `kubectl get cluster -n platform-postgres` shows `Ready` replicas matching `instances`.
- `kubectl logs deploy/cnpg-controller-manager -n cnpg-system` has no failing webhooks.
- `kubectl get podmonitor -n platform-postgres` returns entries for the cluster and pooler.
- `psql "<rw-connection-string>" -c 'select 1'` works from CI runner subnet.
- `cnpg` `barman-cloud-backup-list` shows successful full + WAL backups.
## Offline notes
- Mirror the operator manifest and container images to the approved registry first; no live downloads occur at runtime.
- If Prometheus is not present, leave PodMonitor applied; it is inert without the CRD.

57
devops/database/postgres/cluster-production.yaml Normal file
View File

@@ -0,0 +1,57 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: stellaops-pg-prod
namespace: platform-postgres
spec:
instances: 3
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
primaryUpdateStrategy: unsupervised
storage:
size: 500Gi
storageClass: fast-ssd
walStorage:
size: 128Gi
storageClass: fast-wal
superuserSecret:
name: stellaops-pg-superuser
bootstrap:
initdb:
database: stellaops
owner: stellaops_app
secret:
name: stellaops-pg-app
monitoring:
enablePodMonitor: true
postgresql:
parameters:
max_connections: "900"
shared_buffers: "4096MB"
work_mem: "96MB"
maintenance_work_mem: "768MB"
wal_level: "replica"
max_wal_size: "4GB"
timezone: "UTC"
log_min_duration_statement: "250"
statement_timeout: "30000"
resources:
requests:
cpu: "4"
memory: "16Gi"
limits:
cpu: "8"
memory: "24Gi"
backup:
barmanObjectStore:
destinationPath: s3://stellaops-backups/production
s3Credentials:
accessKeyId:
name: stellaops-pg-backup
key: ACCESS_KEY_ID
secretAccessKey:
name: stellaops-pg-backup
key: SECRET_ACCESS_KEY
wal:
compression: gzip
maxParallel: 4
logLevel: info

57
devops/database/postgres/cluster-staging.yaml Normal file
View File

@@ -0,0 +1,57 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: stellaops-pg-stg
namespace: platform-postgres
spec:
instances: 2
imageName: ghcr.io/cloudnative-pg/postgresql:16.4
primaryUpdateStrategy: unsupervised
storage:
size: 200Gi
storageClass: fast-ssd
walStorage:
size: 64Gi
storageClass: fast-wal
superuserSecret:
name: stellaops-pg-superuser
bootstrap:
initdb:
database: stellaops
owner: stellaops_app
secret:
name: stellaops-pg-app
monitoring:
enablePodMonitor: true
postgresql:
parameters:
max_connections: "600"
shared_buffers: "2048MB"
work_mem: "64MB"
maintenance_work_mem: "512MB"
wal_level: "replica"
max_wal_size: "2GB"
timezone: "UTC"
log_min_duration_statement: "500"
statement_timeout: "30000"
resources:
requests:
cpu: "2"
memory: "8Gi"
limits:
cpu: "4"
memory: "12Gi"
backup:
barmanObjectStore:
destinationPath: s3://stellaops-backups/staging
s3Credentials:
accessKeyId:
name: stellaops-pg-backup
key: ACCESS_KEY_ID
secretAccessKey:
name: stellaops-pg-backup
key: SECRET_ACCESS_KEY
wal:
compression: gzip
maxParallel: 2
logLevel: info

4
devops/database/postgres/namespace.yaml Normal file
View File

@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: platform-postgres

29
devops/database/postgres/pooler-production.yaml Normal file
View File

@@ -0,0 +1,29 @@
apiVersion: postgresql.cnpg.io/v1
kind: Pooler
metadata:
name: stellaops-pg-prod-pooler
namespace: platform-postgres
spec:
cluster:
name: stellaops-pg-prod
instances: 3
type: rw
pgbouncer:
parameters:
pool_mode: transaction
max_client_conn: "1500"
default_pool_size: "80"
server_reset_query: "RESET ALL; SET SESSION AUTHORIZATION DEFAULT; SET TIME ZONE 'UTC';"
authQuerySecret:
name: stellaops-pg-app
template:
spec:
containers:
- name: pgbouncer
resources:
requests:
cpu: "150m"
memory: "192Mi"
limits:
cpu: "750m"
memory: "384Mi"

29
devops/database/postgres/pooler-staging.yaml Normal file
View File

@@ -0,0 +1,29 @@
apiVersion: postgresql.cnpg.io/v1
kind: Pooler
metadata:
name: stellaops-pg-stg-pooler
namespace: platform-postgres
spec:
cluster:
name: stellaops-pg-stg
instances: 2
type: rw
pgbouncer:
parameters:
pool_mode: transaction
max_client_conn: "800"
default_pool_size: "50"
server_reset_query: "RESET ALL; SET SESSION AUTHORIZATION DEFAULT; SET TIME ZONE 'UTC';"
authQuerySecret:
name: stellaops-pg-app
template:
spec:
containers:
- name: pgbouncer
resources:
requests:
cpu: "100m"
memory: "128Mi"
limits:
cpu: "500m"
memory: "256Mi"

9
devops/database/postgres/secrets/example-app.yaml Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: stellaops-pg-app
namespace: platform-postgres
type: kubernetes.io/basic-auth
stringData:
username: stellaops_app
password: CHANGEME_APP_PASSWORD

9
devops/database/postgres/secrets/example-backup-credentials.yaml Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: stellaops-pg-backup
namespace: platform-postgres
type: Opaque
stringData:
ACCESS_KEY_ID: CHANGEME_ACCESS_KEY
SECRET_ACCESS_KEY: CHANGEME_SECRET_KEY

9
devops/database/postgres/secrets/example-superuser.yaml Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: v1
kind: Secret
metadata:
name: stellaops-pg-superuser
namespace: platform-postgres
type: kubernetes.io/basic-auth
stringData:
username: postgres
password: CHANGEME_SUPERUSER_PASSWORD