feat(rate-limiting): Implement core rate limiting functionality with configuration, decision-making, metrics, middleware, and service registration
- Add RateLimitConfig for configuration management with YAML binding support. - Introduce RateLimitDecision to encapsulate the result of rate limit checks. - Implement RateLimitMetrics for OpenTelemetry metrics tracking. - Create RateLimitMiddleware for enforcing rate limits on incoming requests. - Develop RateLimitService to orchestrate instance and environment rate limit checks. - Add RateLimitServiceCollectionExtensions for dependency injection registration.
This commit is contained in:
master
@@ -0,0 +1,117 @@
|
||||
-- Migration: 006_score_replay_tables.sql
|
||||
-- Sprint: SPRINT_3401_0002_0001
|
||||
-- Tasks: SCORE-REPLAY-007 (scan_manifest), SCORE-REPLAY-009 (proof_bundle)
|
||||
-- Description: Tables for score replay and proof bundle functionality
|
||||
|
||||
-- Scan manifests for deterministic replay
|
||||
CREATE TABLE IF NOT EXISTS scan_manifest (
|
||||
manifest_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
scan_id UUID NOT NULL,
|
||||
manifest_hash VARCHAR(128) NOT NULL, -- SHA-256 of manifest content
|
||||
sbom_hash VARCHAR(128) NOT NULL, -- Hash of input SBOM
|
||||
rules_hash VARCHAR(128) NOT NULL, -- Hash of rules snapshot
|
||||
feed_hash VARCHAR(128) NOT NULL, -- Hash of advisory feed snapshot
|
||||
policy_hash VARCHAR(128) NOT NULL, -- Hash of scoring policy
|
||||
|
||||
-- Evidence timing
|
||||
scan_started_at TIMESTAMPTZ NOT NULL,
|
||||
scan_completed_at TIMESTAMPTZ,
|
||||
|
||||
-- Content (stored as JSONB for query flexibility)
|
||||
manifest_content JSONB NOT NULL,
|
||||
|
||||
-- Metadata
|
||||
scanner_version VARCHAR(64) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
-- Constraints
|
||||
CONSTRAINT fk_scan_manifest_scan FOREIGN KEY (scan_id) REFERENCES scans(scan_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for manifest hash lookups (for deduplication and verification)
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_manifest_hash ON scan_manifest(manifest_hash);
|
||||
|
||||
-- Index for scan lookups
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_manifest_scan_id ON scan_manifest(scan_id);
|
||||
|
||||
-- Index for temporal queries
|
||||
CREATE INDEX IF NOT EXISTS idx_scan_manifest_created_at ON scan_manifest(created_at DESC);
|
||||
|
||||
-- Proof bundles for cryptographic evidence chains
|
||||
CREATE TABLE IF NOT EXISTS proof_bundle (
|
||||
scan_id UUID NOT NULL,
|
||||
root_hash VARCHAR(128) NOT NULL, -- Merkle root of all evidence
|
||||
bundle_type VARCHAR(32) NOT NULL DEFAULT 'standard', -- 'standard', 'extended', 'minimal'
|
||||
|
||||
-- DSSE envelope for the bundle
|
||||
dsse_envelope JSONB, -- Full DSSE-signed envelope
|
||||
signature_keyid VARCHAR(256), -- Key ID used for signing
|
||||
signature_algorithm VARCHAR(64), -- e.g., 'ed25519', 'rsa-pss-sha256'
|
||||
|
||||
-- Bundle content
|
||||
bundle_content BYTEA, -- ZIP archive or raw bundle data
|
||||
bundle_hash VARCHAR(128) NOT NULL, -- SHA-256 of bundle_content
|
||||
|
||||
-- Component hashes for incremental verification
|
||||
ledger_hash VARCHAR(128), -- Hash of proof ledger
|
||||
manifest_hash VARCHAR(128), -- Reference to scan_manifest
|
||||
sbom_hash VARCHAR(128),
|
||||
vex_hash VARCHAR(128),
|
||||
|
||||
-- Metadata
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
expires_at TIMESTAMPTZ, -- Optional TTL for retention
|
||||
|
||||
-- Primary key is (scan_id, root_hash) to allow multiple bundles per scan
|
||||
PRIMARY KEY (scan_id, root_hash),
|
||||
|
||||
-- Foreign key
|
||||
CONSTRAINT fk_proof_bundle_scan FOREIGN KEY (scan_id) REFERENCES scans(scan_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for root hash lookups (for verification)
|
||||
CREATE INDEX IF NOT EXISTS idx_proof_bundle_root_hash ON proof_bundle(root_hash);
|
||||
|
||||
-- Index for temporal queries
|
||||
CREATE INDEX IF NOT EXISTS idx_proof_bundle_created_at ON proof_bundle(created_at DESC);
|
||||
|
||||
-- Index for expiration cleanup
|
||||
CREATE INDEX IF NOT EXISTS idx_proof_bundle_expires_at ON proof_bundle(expires_at) WHERE expires_at IS NOT NULL;
|
||||
|
||||
-- Score replay history for tracking rescores
|
||||
CREATE TABLE IF NOT EXISTS score_replay_history (
|
||||
replay_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
scan_id UUID NOT NULL,
|
||||
|
||||
-- What triggered the replay
|
||||
trigger_type VARCHAR(32) NOT NULL, -- 'feed_update', 'policy_change', 'manual', 'scheduled'
|
||||
trigger_reference VARCHAR(256), -- Feed snapshot ID, policy version, etc.
|
||||
|
||||
-- Before/after state
|
||||
original_manifest_hash VARCHAR(128),
|
||||
replayed_manifest_hash VARCHAR(128),
|
||||
|
||||
-- Score delta summary
|
||||
score_delta_json JSONB, -- Summary of changed scores
|
||||
findings_added INT DEFAULT 0,
|
||||
findings_removed INT DEFAULT 0,
|
||||
findings_rescored INT DEFAULT 0,
|
||||
|
||||
-- Timing
|
||||
replayed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
duration_ms INT,
|
||||
|
||||
-- Foreign key
|
||||
CONSTRAINT fk_score_replay_scan FOREIGN KEY (scan_id) REFERENCES scans(scan_id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- Index for scan-based lookups
|
||||
CREATE INDEX IF NOT EXISTS idx_score_replay_scan_id ON score_replay_history(scan_id);
|
||||
|
||||
-- Index for temporal queries
|
||||
CREATE INDEX IF NOT EXISTS idx_score_replay_replayed_at ON score_replay_history(replayed_at DESC);
|
||||
|
||||
-- Comments for documentation
|
||||
COMMENT ON TABLE scan_manifest IS 'Deterministic scan manifests for score replay. Each manifest captures all inputs needed to reproduce a scan result.';
|
||||
COMMENT ON TABLE proof_bundle IS 'Cryptographically-signed evidence bundles for audit trails. Contains DSSE-wrapped proof chains.';
|
||||
COMMENT ON TABLE score_replay_history IS 'History of score replays triggered by feed updates, policy changes, or manual requests.';
|
||||
@@ -0,0 +1,64 @@
|
||||
-- Migration: 007_unknowns_ranking_containment.sql
|
||||
-- Sprint: SPRINT_3600_0002_0001
|
||||
-- Task: UNK-RANK-005 - Add blast_radius, containment columns to unknowns table
|
||||
-- Description: Extend unknowns table with ranking signals for containment-aware scoring
|
||||
|
||||
-- Add blast radius columns
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS blast_dependents INT DEFAULT 0;
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS blast_net_facing BOOLEAN DEFAULT false;
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS blast_privilege TEXT DEFAULT 'user';
|
||||
|
||||
-- Add exploit pressure columns
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS epss DOUBLE PRECISION;
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS kev BOOLEAN DEFAULT false;
|
||||
|
||||
-- Add containment signal columns
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS containment_seccomp TEXT DEFAULT 'unknown';
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS containment_fs TEXT DEFAULT 'unknown';
|
||||
|
||||
-- Add proof reference for ranking explanation
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS proof_ref TEXT;
|
||||
|
||||
-- Add evidence scarcity column (0-1 range)
|
||||
ALTER TABLE unknowns ADD COLUMN IF NOT EXISTS evidence_scarcity DOUBLE PRECISION DEFAULT 0.5;
|
||||
|
||||
-- Update score index for efficient sorting
|
||||
DROP INDEX IF EXISTS ix_unknowns_score_desc;
|
||||
CREATE INDEX IF NOT EXISTS ix_unknowns_score_desc ON unknowns(score DESC);
|
||||
|
||||
-- Composite index for common query patterns
|
||||
DROP INDEX IF EXISTS ix_unknowns_artifact_score;
|
||||
CREATE INDEX IF NOT EXISTS ix_unknowns_artifact_score ON unknowns(artifact_digest, score DESC);
|
||||
|
||||
-- Index for filtering by containment state
|
||||
DROP INDEX IF EXISTS ix_unknowns_containment;
|
||||
CREATE INDEX IF NOT EXISTS ix_unknowns_containment ON unknowns(containment_seccomp, containment_fs);
|
||||
|
||||
-- Index for KEV filtering (high priority unknowns)
|
||||
DROP INDEX IF EXISTS ix_unknowns_kev;
|
||||
CREATE INDEX IF NOT EXISTS ix_unknowns_kev ON unknowns(kev) WHERE kev = true;
|
||||
|
||||
-- Comments for documentation
|
||||
COMMENT ON COLUMN unknowns.blast_dependents IS 'Number of dependent packages affected by this unknown';
|
||||
COMMENT ON COLUMN unknowns.blast_net_facing IS 'Whether the affected code is network-facing';
|
||||
COMMENT ON COLUMN unknowns.blast_privilege IS 'Privilege level: root, user, unprivileged';
|
||||
COMMENT ON COLUMN unknowns.epss IS 'EPSS score if available (0.0-1.0)';
|
||||
COMMENT ON COLUMN unknowns.kev IS 'True if vulnerability is in CISA KEV catalog';
|
||||
COMMENT ON COLUMN unknowns.containment_seccomp IS 'Seccomp state: enforced, permissive, unknown';
|
||||
COMMENT ON COLUMN unknowns.containment_fs IS 'Filesystem state: ro (read-only), rw, unknown';
|
||||
COMMENT ON COLUMN unknowns.proof_ref IS 'Path to proof bundle explaining ranking factors';
|
||||
COMMENT ON COLUMN unknowns.evidence_scarcity IS 'Evidence scarcity factor (0=full evidence, 1=no evidence)';
|
||||
|
||||
-- Check constraint for valid privilege values
|
||||
ALTER TABLE unknowns DROP CONSTRAINT IF EXISTS chk_unknowns_privilege;
|
||||
ALTER TABLE unknowns ADD CONSTRAINT chk_unknowns_privilege
|
||||
CHECK (blast_privilege IN ('root', 'user', 'unprivileged'));
|
||||
|
||||
-- Check constraint for valid containment values
|
||||
ALTER TABLE unknowns DROP CONSTRAINT IF EXISTS chk_unknowns_seccomp;
|
||||
ALTER TABLE unknowns ADD CONSTRAINT chk_unknowns_seccomp
|
||||
CHECK (containment_seccomp IN ('enforced', 'permissive', 'unknown'));
|
||||
|
||||
ALTER TABLE unknowns DROP CONSTRAINT IF EXISTS chk_unknowns_fs;
|
||||
ALTER TABLE unknowns ADD CONSTRAINT chk_unknowns_fs
|
||||
CHECK (containment_fs IN ('ro', 'rw', 'unknown'));
|
||||
@@ -0,0 +1,292 @@
|
||||
-- SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
-- Sprint: Advisory-derived
|
||||
-- Task: EPSS Integration - Database Schema
|
||||
-- Description: Creates tables for EPSS (Exploit Prediction Scoring System) integration
|
||||
-- with time-series storage and change detection
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Import Provenance
|
||||
-- ============================================================================
|
||||
-- Tracks all EPSS import runs with full provenance for audit and replay
|
||||
CREATE TABLE IF NOT EXISTS epss_import_runs (
|
||||
import_run_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
model_date DATE NOT NULL,
|
||||
source_uri TEXT NOT NULL,
|
||||
retrieved_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
file_sha256 TEXT NOT NULL,
|
||||
decompressed_sha256 TEXT,
|
||||
row_count INT NOT NULL,
|
||||
model_version_tag TEXT, -- e.g., v2025.03.14 from leading # comment
|
||||
published_date DATE, -- from leading # comment if present
|
||||
status TEXT NOT NULL CHECK (status IN ('PENDING', 'SUCCEEDED', 'FAILED')),
|
||||
error TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT epss_import_runs_model_date_unique UNIQUE (model_date)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_import_runs_model_date
|
||||
ON epss_import_runs (model_date DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_import_runs_status
|
||||
ON epss_import_runs (status);
|
||||
|
||||
COMMENT ON TABLE epss_import_runs IS 'Provenance tracking for all EPSS import operations';
|
||||
COMMENT ON COLUMN epss_import_runs.model_date IS 'The date of the EPSS model snapshot';
|
||||
COMMENT ON COLUMN epss_import_runs.source_uri IS 'Source URL or bundle:// URI for the import';
|
||||
COMMENT ON COLUMN epss_import_runs.file_sha256 IS 'SHA256 hash of the compressed file';
|
||||
COMMENT ON COLUMN epss_import_runs.decompressed_sha256 IS 'SHA256 hash of the decompressed CSV';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Time-Series Scores (Partitioned)
|
||||
-- ============================================================================
|
||||
-- Immutable append-only storage for all EPSS scores by date
|
||||
-- Partitioned by month for efficient querying and maintenance
|
||||
CREATE TABLE IF NOT EXISTS epss_scores (
|
||||
model_date DATE NOT NULL,
|
||||
cve_id TEXT NOT NULL,
|
||||
epss_score DOUBLE PRECISION NOT NULL CHECK (epss_score >= 0 AND epss_score <= 1),
|
||||
percentile DOUBLE PRECISION NOT NULL CHECK (percentile >= 0 AND percentile <= 1),
|
||||
import_run_id UUID NOT NULL REFERENCES epss_import_runs(import_run_id),
|
||||
PRIMARY KEY (model_date, cve_id)
|
||||
) PARTITION BY RANGE (model_date);
|
||||
|
||||
-- Create partitions for current and next 6 months
|
||||
-- Additional partitions should be created via scheduled maintenance
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2025_12 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2025-12-01') TO ('2026-01-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_01 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-01-01') TO ('2026-02-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_02 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-02-01') TO ('2026-03-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_03 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-03-01') TO ('2026-04-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_04 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-04-01') TO ('2026-05-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_2026_05 PARTITION OF epss_scores
|
||||
FOR VALUES FROM ('2026-05-01') TO ('2026-06-01');
|
||||
|
||||
-- Default partition for dates outside defined ranges
|
||||
CREATE TABLE IF NOT EXISTS epss_scores_default PARTITION OF epss_scores DEFAULT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_scores_cve_id
|
||||
ON epss_scores (cve_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_scores_score_desc
|
||||
ON epss_scores (epss_score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_scores_cve_date
|
||||
ON epss_scores (cve_id, model_date DESC);
|
||||
|
||||
COMMENT ON TABLE epss_scores IS 'Immutable time-series storage for all EPSS scores';
|
||||
COMMENT ON COLUMN epss_scores.epss_score IS 'EPSS probability score (0.0 to 1.0)';
|
||||
COMMENT ON COLUMN epss_scores.percentile IS 'Percentile rank vs all CVEs (0.0 to 1.0)';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Current Projection (Fast Lookup)
|
||||
-- ============================================================================
|
||||
-- Materialized current EPSS for fast O(1) lookup
|
||||
-- Updated during each import after delta computation
|
||||
CREATE TABLE IF NOT EXISTS epss_current (
|
||||
cve_id TEXT PRIMARY KEY,
|
||||
epss_score DOUBLE PRECISION NOT NULL CHECK (epss_score >= 0 AND epss_score <= 1),
|
||||
percentile DOUBLE PRECISION NOT NULL CHECK (percentile >= 0 AND percentile <= 1),
|
||||
model_date DATE NOT NULL,
|
||||
import_run_id UUID NOT NULL REFERENCES epss_import_runs(import_run_id),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_current_score_desc
|
||||
ON epss_current (epss_score DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_current_percentile_desc
|
||||
ON epss_current (percentile DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_current_model_date
|
||||
ON epss_current (model_date);
|
||||
|
||||
COMMENT ON TABLE epss_current IS 'Fast lookup projection of latest EPSS scores';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Change Detection (Partitioned)
|
||||
-- ============================================================================
|
||||
-- Tracks daily changes to enable efficient targeted enrichment
|
||||
CREATE TABLE IF NOT EXISTS epss_changes (
|
||||
model_date DATE NOT NULL,
|
||||
cve_id TEXT NOT NULL,
|
||||
old_score DOUBLE PRECISION,
|
||||
new_score DOUBLE PRECISION NOT NULL,
|
||||
delta_score DOUBLE PRECISION,
|
||||
old_percentile DOUBLE PRECISION,
|
||||
new_percentile DOUBLE PRECISION NOT NULL,
|
||||
delta_percentile DOUBLE PRECISION,
|
||||
flags INT NOT NULL DEFAULT 0,
|
||||
import_run_id UUID NOT NULL REFERENCES epss_import_runs(import_run_id),
|
||||
PRIMARY KEY (model_date, cve_id)
|
||||
) PARTITION BY RANGE (model_date);
|
||||
|
||||
-- Create partitions matching epss_scores
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2025_12 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2025-12-01') TO ('2026-01-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_01 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-01-01') TO ('2026-02-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_02 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-02-01') TO ('2026-03-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_03 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-03-01') TO ('2026-04-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_04 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-04-01') TO ('2026-05-01');
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_2026_05 PARTITION OF epss_changes
|
||||
FOR VALUES FROM ('2026-05-01') TO ('2026-06-01');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS epss_changes_default PARTITION OF epss_changes DEFAULT;
|
||||
|
||||
-- Flags bitmask values:
|
||||
-- 0x01 = NEW_SCORED (CVE newly scored)
|
||||
-- 0x02 = CROSSED_HIGH (crossed above high score threshold)
|
||||
-- 0x04 = CROSSED_LOW (crossed below high score threshold)
|
||||
-- 0x08 = BIG_JUMP_UP (delta > 0.10 upward)
|
||||
-- 0x10 = BIG_JUMP_DOWN (delta > 0.10 downward)
|
||||
-- 0x20 = TOP_PERCENTILE (entered top 5%)
|
||||
-- 0x40 = LEFT_TOP_PERCENTILE (left top 5%)
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_changes_flags
|
||||
ON epss_changes (flags) WHERE flags > 0;
|
||||
CREATE INDEX IF NOT EXISTS idx_epss_changes_delta
|
||||
ON epss_changes (ABS(delta_score) DESC) WHERE delta_score IS NOT NULL;
|
||||
|
||||
COMMENT ON TABLE epss_changes IS 'Daily change detection for targeted enrichment';
|
||||
COMMENT ON COLUMN epss_changes.flags IS 'Bitmask: 0x01=NEW, 0x02=CROSSED_HIGH, 0x04=CROSSED_LOW, 0x08=BIG_UP, 0x10=BIG_DOWN, 0x20=TOP_PCT';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Configuration
|
||||
-- ============================================================================
|
||||
-- Per-org or global thresholds for notification and scoring
|
||||
CREATE TABLE IF NOT EXISTS epss_config (
|
||||
config_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
org_id UUID, -- NULL for global defaults
|
||||
high_percentile DOUBLE PRECISION NOT NULL DEFAULT 0.95,
|
||||
high_score DOUBLE PRECISION NOT NULL DEFAULT 0.50,
|
||||
big_jump_delta DOUBLE PRECISION NOT NULL DEFAULT 0.10,
|
||||
score_weight DOUBLE PRECISION NOT NULL DEFAULT 0.25,
|
||||
notify_on_new_high BOOLEAN NOT NULL DEFAULT true,
|
||||
notify_on_crossing BOOLEAN NOT NULL DEFAULT true,
|
||||
notify_on_big_jump BOOLEAN NOT NULL DEFAULT true,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT epss_config_org_unique UNIQUE (org_id)
|
||||
);
|
||||
|
||||
-- Insert global defaults
|
||||
INSERT INTO epss_config (org_id, high_percentile, high_score, big_jump_delta, score_weight)
|
||||
VALUES (NULL, 0.95, 0.50, 0.10, 0.25)
|
||||
ON CONFLICT (org_id) DO NOTHING;
|
||||
|
||||
COMMENT ON TABLE epss_config IS 'EPSS notification and scoring thresholds';
|
||||
COMMENT ON COLUMN epss_config.high_percentile IS 'Threshold for top percentile alerts (default: 0.95 = top 5%)';
|
||||
COMMENT ON COLUMN epss_config.high_score IS 'Threshold for high score alerts (default: 0.50)';
|
||||
COMMENT ON COLUMN epss_config.big_jump_delta IS 'Threshold for significant daily change (default: 0.10)';
|
||||
|
||||
-- ============================================================================
|
||||
-- EPSS Evidence on Scan Findings
|
||||
-- ============================================================================
|
||||
-- Add EPSS-at-scan columns to existing scan_findings if not exists
|
||||
-- This preserves immutable evidence for replay
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'scan_findings' AND column_name = 'epss_score_at_scan'
|
||||
) THEN
|
||||
ALTER TABLE scan_findings
|
||||
ADD COLUMN epss_score_at_scan DOUBLE PRECISION,
|
||||
ADD COLUMN epss_percentile_at_scan DOUBLE PRECISION,
|
||||
ADD COLUMN epss_model_date_at_scan DATE,
|
||||
ADD COLUMN epss_import_run_id_at_scan UUID;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- ============================================================================
|
||||
-- Helper Functions
|
||||
-- ============================================================================
|
||||
|
||||
-- Function to compute change flags
|
||||
CREATE OR REPLACE FUNCTION compute_epss_change_flags(
|
||||
p_old_score DOUBLE PRECISION,
|
||||
p_new_score DOUBLE PRECISION,
|
||||
p_old_percentile DOUBLE PRECISION,
|
||||
p_new_percentile DOUBLE PRECISION,
|
||||
p_high_score DOUBLE PRECISION DEFAULT 0.50,
|
||||
p_high_percentile DOUBLE PRECISION DEFAULT 0.95,
|
||||
p_big_jump DOUBLE PRECISION DEFAULT 0.10
|
||||
) RETURNS INT AS $$
|
||||
DECLARE
|
||||
v_flags INT := 0;
|
||||
v_delta DOUBLE PRECISION;
|
||||
BEGIN
|
||||
-- NEW_SCORED
|
||||
IF p_old_score IS NULL THEN
|
||||
v_flags := v_flags | 1; -- 0x01
|
||||
END IF;
|
||||
|
||||
-- CROSSED_HIGH (score)
|
||||
IF p_old_score IS NOT NULL AND p_old_score < p_high_score AND p_new_score >= p_high_score THEN
|
||||
v_flags := v_flags | 2; -- 0x02
|
||||
END IF;
|
||||
|
||||
-- CROSSED_LOW (score)
|
||||
IF p_old_score IS NOT NULL AND p_old_score >= p_high_score AND p_new_score < p_high_score THEN
|
||||
v_flags := v_flags | 4; -- 0x04
|
||||
END IF;
|
||||
|
||||
-- BIG_JUMP_UP
|
||||
IF p_old_score IS NOT NULL THEN
|
||||
v_delta := p_new_score - p_old_score;
|
||||
IF v_delta > p_big_jump THEN
|
||||
v_flags := v_flags | 8; -- 0x08
|
||||
END IF;
|
||||
|
||||
-- BIG_JUMP_DOWN
|
||||
IF v_delta < -p_big_jump THEN
|
||||
v_flags := v_flags | 16; -- 0x10
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
-- TOP_PERCENTILE (entered)
|
||||
IF (p_old_percentile IS NULL OR p_old_percentile < p_high_percentile)
|
||||
AND p_new_percentile >= p_high_percentile THEN
|
||||
v_flags := v_flags | 32; -- 0x20
|
||||
END IF;
|
||||
|
||||
-- LEFT_TOP_PERCENTILE
|
||||
IF p_old_percentile IS NOT NULL AND p_old_percentile >= p_high_percentile
|
||||
AND p_new_percentile < p_high_percentile THEN
|
||||
v_flags := v_flags | 64; -- 0x40
|
||||
END IF;
|
||||
|
||||
RETURN v_flags;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
||||
|
||||
COMMENT ON FUNCTION compute_epss_change_flags IS 'Computes bitmask flags for EPSS change detection';
|
||||
|
||||
-- Function to create monthly partition
|
||||
CREATE OR REPLACE FUNCTION create_epss_partition(p_year INT, p_month INT)
|
||||
RETURNS VOID AS $$
|
||||
DECLARE
|
||||
v_start DATE;
|
||||
v_end DATE;
|
||||
v_partition_name TEXT;
|
||||
BEGIN
|
||||
v_start := make_date(p_year, p_month, 1);
|
||||
v_end := v_start + INTERVAL '1 month';
|
||||
v_partition_name := format('epss_scores_%s_%s', p_year, LPAD(p_month::TEXT, 2, '0'));
|
||||
|
||||
EXECUTE format(
|
||||
'CREATE TABLE IF NOT EXISTS %I PARTITION OF epss_scores FOR VALUES FROM (%L) TO (%L)',
|
||||
v_partition_name, v_start, v_end
|
||||
);
|
||||
|
||||
v_partition_name := format('epss_changes_%s_%s', p_year, LPAD(p_month::TEXT, 2, '0'));
|
||||
EXECUTE format(
|
||||
'CREATE TABLE IF NOT EXISTS %I PARTITION OF epss_changes FOR VALUES FROM (%L) TO (%L)',
|
||||
v_partition_name, v_start, v_end
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
COMMENT ON FUNCTION create_epss_partition IS 'Creates monthly partitions for EPSS tables';
|
||||
@@ -6,4 +6,8 @@ internal static class MigrationIds
|
||||
public const string ProofSpineTables = "002_proof_spine_tables.sql";
|
||||
public const string ClassificationHistory = "003_classification_history.sql";
|
||||
public const string ScanMetrics = "004_scan_metrics.sql";
|
||||
public const string SmartDiffTables = "005_smart_diff_tables.sql";
|
||||
public const string ScoreReplayTables = "006_score_replay_tables.sql";
|
||||
public const string UnknownsRankingContainment = "007_unknowns_ranking_containment.sql";
|
||||
public const string EpssIntegration = "008_epss_integration.sql";
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user