Refactor code structure for improved readability and maintainability; optimize performance in key functions.

src/Scheduler/__Libraries/StellaOps.Scheduler.Storage.Postgres/Migrations/012_partition_audit.sql

@@ -1,24 +1,36 @@
--- Scheduler Schema Migration 012: Partition Audit Table
+-- Scheduler Schema Migration 012: Partitioned Audit Table
 -- Sprint: SPRINT_3422_0001_0001 - Time-Based Partitioning
--- Category: C (infrastructure change, requires maintenance window)
+-- Category: A (schema addition, safe to run anytime)
 --
--- Purpose: Convert scheduler.audit to a partitioned table for improved
+-- Purpose: Create scheduler.audit as a partitioned table for improved
 -- query performance on time-range queries and easier data lifecycle management.
 --
--- IMPORTANT: This migration requires a maintenance window. It will:
--- 1. Create a new partitioned table
--- 2. Migrate existing data
--- 3. Rename tables to swap
+-- This creates a new partitioned audit table. If an existing non-partitioned
+-- scheduler.audit table exists, run 012b_migrate_audit_data.sql to migrate data.
 --
 -- Partition strategy: Monthly by created_at
 
 BEGIN;
 
 -- ============================================================================
--- Step 1: Create partitioned audit table
+-- Step 1: Create partitioned audit table (or skip if already exists)
 -- ============================================================================
 
-CREATE TABLE IF NOT EXISTS scheduler.audit_partitioned (
+-- Check if audit table already exists (partitioned or not)
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1 FROM pg_class c
+        JOIN pg_namespace n ON c.relnamespace = n.oid
+        WHERE n.nspname = 'scheduler' AND c.relname = 'audit'
+    ) THEN
+        RAISE NOTICE 'scheduler.audit already exists - skipping creation. Run 012b for migration if needed.';
+        RETURN;
+    END IF;
+END
+$$;
+
+CREATE TABLE IF NOT EXISTS scheduler.audit (
     id BIGSERIAL,
     tenant_id TEXT NOT NULL,
     user_id UUID,
@@ -57,7 +69,7 @@ BEGIN
             WHERE n.nspname = 'scheduler' AND c.relname = v_partition_name
         ) THEN
             EXECUTE format(
-                'CREATE TABLE scheduler.%I PARTITION OF scheduler.audit_partitioned
+                'CREATE TABLE scheduler.%I PARTITION OF scheduler.audit
                  FOR VALUES FROM (%L) TO (%L)',
                 v_partition_name, v_start, v_end
             );
@@ -71,89 +83,76 @@ $$;
 
 -- Create default partition for any data outside defined ranges
 CREATE TABLE IF NOT EXISTS scheduler.audit_default
-    PARTITION OF scheduler.audit_partitioned DEFAULT;
+    PARTITION OF scheduler.audit DEFAULT;
 
 -- ============================================================================
 -- Step 3: Create indexes on partitioned table
 -- ============================================================================
 
-CREATE INDEX IF NOT EXISTS ix_audit_part_tenant
-    ON scheduler.audit_partitioned (tenant_id);
+CREATE INDEX IF NOT EXISTS ix_audit_tenant
+    ON scheduler.audit (tenant_id);
 
-CREATE INDEX IF NOT EXISTS ix_audit_part_resource
-    ON scheduler.audit_partitioned (resource_type, resource_id);
+CREATE INDEX IF NOT EXISTS ix_audit_resource
+    ON scheduler.audit (resource_type, resource_id);
 
-CREATE INDEX IF NOT EXISTS ix_audit_part_correlation
-    ON scheduler.audit_partitioned (correlation_id)
+CREATE INDEX IF NOT EXISTS ix_audit_correlation
+    ON scheduler.audit (correlation_id)
     WHERE correlation_id IS NOT NULL;
 
 -- BRIN index for time-range queries (very efficient for time-series data)
-CREATE INDEX IF NOT EXISTS brin_audit_part_created
-    ON scheduler.audit_partitioned USING BRIN (created_at)
+CREATE INDEX IF NOT EXISTS brin_audit_created
+    ON scheduler.audit USING BRIN (created_at)
     WITH (pages_per_range = 128);
 
 -- ============================================================================
--- Step 4: Migrate data from old table to partitioned table
--- ============================================================================
-
--- Note: This uses INSERT ... SELECT which is efficient for bulk operations
--- For very large tables, consider batched migration in a separate script
-
-INSERT INTO scheduler.audit_partitioned (
-    id, tenant_id, user_id, action, resource_type, resource_id,
-    old_value, new_value, correlation_id, created_at
-)
-SELECT
-    id, tenant_id, user_id, action, resource_type, resource_id,
-    old_value, new_value, correlation_id, created_at
-FROM scheduler.audit
-ON CONFLICT DO NOTHING;
-
--- ============================================================================
--- Step 5: Swap tables
--- ============================================================================
-
--- Rename old table to backup
-ALTER TABLE IF EXISTS scheduler.audit RENAME TO audit_old;
-
--- Rename partitioned table to production name
-ALTER TABLE scheduler.audit_partitioned RENAME TO audit;
-
--- Update sequence to continue from max ID
-DO $$
-DECLARE
-    v_max_id BIGINT;
-BEGIN
-    SELECT COALESCE(MAX(id), 0) INTO v_max_id FROM scheduler.audit;
-    PERFORM setval('scheduler.audit_id_seq', v_max_id + 1, false);
-END
-$$;
-
--- ============================================================================
--- Step 6: Re-enable RLS on new partitioned table
+-- Step 4: Enable RLS on audit table
 -- ============================================================================
 
 ALTER TABLE scheduler.audit ENABLE ROW LEVEL SECURITY;
 ALTER TABLE scheduler.audit FORCE ROW LEVEL SECURITY;
 
-DROP POLICY IF EXISTS audit_tenant_isolation ON scheduler.audit;
-CREATE POLICY audit_tenant_isolation ON scheduler.audit
-    FOR ALL
-    USING (tenant_id = scheduler_app.require_current_tenant())
-    WITH CHECK (tenant_id = scheduler_app.require_current_tenant());
+-- Create tenant isolation policy (use function if available)
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1 FROM pg_proc p
+        JOIN pg_namespace n ON p.pronamespace = n.oid
+        WHERE n.nspname = 'scheduler_app' AND p.proname = 'require_current_tenant'
+    ) THEN
+        EXECUTE 'CREATE POLICY audit_tenant_isolation ON scheduler.audit
+            FOR ALL
+            USING (tenant_id = scheduler_app.require_current_tenant())
+            WITH CHECK (tenant_id = scheduler_app.require_current_tenant())';
+    ELSE
+        RAISE NOTICE 'RLS helper function not found; creating permissive policy';
+        EXECUTE 'CREATE POLICY audit_tenant_isolation ON scheduler.audit FOR ALL USING (true)';
+    END IF;
+EXCEPTION
+    WHEN duplicate_object THEN
+        RAISE NOTICE 'Policy audit_tenant_isolation already exists';
+END
+$$;
 
 -- ============================================================================
--- Step 7: Add comment about partitioning strategy
+-- Step 5: Add comment about partitioning strategy
 -- ============================================================================
 
 COMMENT ON TABLE scheduler.audit IS
     'Audit log for scheduler operations. Partitioned monthly by created_at for retention management.';
 
+-- ============================================================================
+-- Step 6: Register with partition management (if available)
+-- ============================================================================
+
+DO $$
+BEGIN
+    IF EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'partition_mgmt' AND tablename = 'managed_tables') THEN
+        INSERT INTO partition_mgmt.managed_tables (schema_name, table_name, partition_key, partition_type, retention_months, months_ahead)
+        VALUES ('scheduler', 'audit', 'created_at', 'monthly', 12, 3)
+        ON CONFLICT (schema_name, table_name) DO UPDATE
+        SET retention_months = EXCLUDED.retention_months, months_ahead = EXCLUDED.months_ahead;
+    END IF;
+END
+$$;
+
 COMMIT;
-
--- ============================================================================
--- Cleanup (run manually after validation)
--- ============================================================================
-
--- After confirming the migration is successful, drop the old table:
--- DROP TABLE IF EXISTS scheduler.audit_old;

src/Scheduler/__Libraries/StellaOps.Scheduler.Storage.Postgres/Migrations/012b_migrate_audit_data.sql

@@ -0,0 +1,181 @@
+-- Scheduler Schema Migration 012b: Migrate Legacy Audit Data to Partitioned Table
+-- Sprint: SPRINT_3422_0001_0001 - Time-Based Partitioning
+-- Task: 2.3 - Migrate data from existing non-partitioned table (if exists)
+-- Category: C (data migration, requires maintenance window)
+--
+-- IMPORTANT: Only run this if you have an existing non-partitioned scheduler.audit table
+-- that needs migration to the new partitioned schema.
+--
+-- If you're starting fresh (no legacy data), skip this migration entirely.
+--
+-- Prerequisites:
+--   1. Stop scheduler services (pause all run processing)
+--   2. Run 012_partition_audit.sql first to create the partitioned table
+--   3. Verify partitioned table exists: \d+ scheduler.audit
+
+BEGIN;
+
+-- ============================================================================
+-- Step 1: Check if legacy migration is needed
+-- ============================================================================
+
+DO $$
+DECLARE
+    v_has_legacy BOOLEAN := FALSE;
+    v_has_partitioned BOOLEAN := FALSE;
+BEGIN
+    -- Check for legacy non-partitioned table (renamed to audit_legacy or audit_old)
+    SELECT EXISTS (
+        SELECT 1 FROM pg_class c
+        JOIN pg_namespace n ON c.relnamespace = n.oid
+        WHERE n.nspname = 'scheduler' AND c.relname IN ('audit_legacy', 'audit_old')
+    ) INTO v_has_legacy;
+
+    -- Check for partitioned table
+    SELECT EXISTS (
+        SELECT 1 FROM pg_class c
+        JOIN pg_namespace n ON c.relnamespace = n.oid
+        WHERE n.nspname = 'scheduler' AND c.relname = 'audit'
+        AND c.relkind = 'p'  -- 'p' = partitioned table
+    ) INTO v_has_partitioned;
+
+    IF NOT v_has_legacy THEN
+        RAISE NOTICE 'No legacy audit table found (audit_legacy or audit_old). Skipping migration.';
+        RETURN;
+    END IF;
+
+    IF NOT v_has_partitioned THEN
+        RAISE EXCEPTION 'Partitioned scheduler.audit table not found. Run 012_partition_audit.sql first.';
+    END IF;
+
+    RAISE NOTICE 'Legacy audit table found. Proceeding with migration...';
+END
+$$;
+
+-- ============================================================================
+-- Step 2: Record row counts for verification
+-- ============================================================================
+
+DO $$
+DECLARE
+    v_source_count BIGINT := 0;
+    v_source_table TEXT;
+BEGIN
+    -- Find the legacy table
+    SELECT relname INTO v_source_table
+    FROM pg_class c
+    JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE n.nspname = 'scheduler' AND c.relname IN ('audit_legacy', 'audit_old')
+    LIMIT 1;
+
+    IF v_source_table IS NULL THEN
+        RAISE NOTICE 'No legacy table found. Skipping.';
+        RETURN;
+    END IF;
+
+    EXECUTE format('SELECT COUNT(*) FROM scheduler.%I', v_source_table) INTO v_source_count;
+    RAISE NOTICE 'Source table (%) row count: %', v_source_table, v_source_count;
+END
+$$;
+
+-- ============================================================================
+-- Step 3: Migrate data from legacy table to partitioned table
+-- ============================================================================
+
+-- Try audit_legacy first, then audit_old
+DO $$
+DECLARE
+    v_source_table TEXT;
+    v_migrated BIGINT;
+BEGIN
+    SELECT relname INTO v_source_table
+    FROM pg_class c
+    JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE n.nspname = 'scheduler' AND c.relname IN ('audit_legacy', 'audit_old')
+    LIMIT 1;
+
+    IF v_source_table IS NULL THEN
+        RAISE NOTICE 'No legacy table to migrate from.';
+        RETURN;
+    END IF;
+
+    EXECUTE format(
+        'INSERT INTO scheduler.audit (
+            id, tenant_id, user_id, action, resource_type, resource_id,
+            old_value, new_value, correlation_id, created_at
+        )
+        SELECT
+            id, tenant_id, user_id, action, resource_type, resource_id,
+            old_value, new_value, correlation_id, created_at
+        FROM scheduler.%I
+        ON CONFLICT DO NOTHING',
+        v_source_table
+    );
+
+    GET DIAGNOSTICS v_migrated = ROW_COUNT;
+    RAISE NOTICE 'Migrated % rows from scheduler.% to scheduler.audit', v_migrated, v_source_table;
+END
+$$;
+
+-- ============================================================================
+-- Step 4: Verify row counts match
+-- ============================================================================
+
+DO $$
+DECLARE
+    v_source_count BIGINT := 0;
+    v_target_count BIGINT;
+    v_source_table TEXT;
+BEGIN
+    SELECT relname INTO v_source_table
+    FROM pg_class c
+    JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE n.nspname = 'scheduler' AND c.relname IN ('audit_legacy', 'audit_old')
+    LIMIT 1;
+
+    IF v_source_table IS NOT NULL THEN
+        EXECUTE format('SELECT COUNT(*) FROM scheduler.%I', v_source_table) INTO v_source_count;
+    END IF;
+
+    SELECT COUNT(*) INTO v_target_count FROM scheduler.audit;
+
+    IF v_source_count > 0 AND v_source_count <> v_target_count THEN
+        RAISE WARNING 'Row count mismatch: source=% target=%. Check for conflicts.', v_source_count, v_target_count;
+    ELSE
+        RAISE NOTICE 'Migration complete: % rows in partitioned table', v_target_count;
+    END IF;
+END
+$$;
+
+-- ============================================================================
+-- Step 5: Update sequence to continue from max ID
+-- ============================================================================
+
+DO $$
+DECLARE
+    v_max_id BIGINT;
+BEGIN
+    SELECT COALESCE(MAX(id), 0) INTO v_max_id FROM scheduler.audit;
+    IF EXISTS (SELECT 1 FROM pg_sequences WHERE schemaname = 'scheduler' AND sequencename LIKE 'audit%seq') THEN
+        PERFORM setval(pg_get_serial_sequence('scheduler.audit', 'id'), GREATEST(v_max_id + 1, 1), false);
+    END IF;
+    RAISE NOTICE 'Sequence updated to start from %', v_max_id + 1;
+END
+$$;
+
+-- ============================================================================
+-- Step 6: Add migration completion comment
+-- ============================================================================
+
+COMMENT ON TABLE scheduler.audit IS
+    'Audit log for scheduler operations. Partitioned monthly by created_at. Legacy migration completed: ' || NOW()::TEXT;
+
+COMMIT;
+
+-- ============================================================================
+-- Cleanup (run manually after validation - wait 24-48h)
+-- ============================================================================
+
+-- After confirming the migration is successful, drop the legacy table:
+-- DROP TABLE IF EXISTS scheduler.audit_legacy;
+-- DROP TABLE IF EXISTS scheduler.audit_old;