fixes save
This commit is contained in:
StellaOps Bot
260
.gitea/scripts/validate/validate-migrations.sh
Normal file
260
.gitea/scripts/validate/validate-migrations.sh
Normal file
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env bash
|
||||
# Migration Validation Script
|
||||
# Validates migration naming conventions, detects duplicates, and checks for issues.
|
||||
#
|
||||
# Usage:
|
||||
# ./validate-migrations.sh [--strict] [--fix-scanner]
|
||||
#
|
||||
# Options:
|
||||
# --strict Exit with error on any warning
|
||||
# --fix-scanner Generate rename commands for Scanner duplicates
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
|
||||
STRICT_MODE=false
|
||||
FIX_SCANNER=false
|
||||
EXIT_CODE=0
|
||||
|
||||
# Parse arguments
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--strict)
|
||||
STRICT_MODE=true
|
||||
shift
|
||||
;;
|
||||
--fix-scanner)
|
||||
FIX_SCANNER=true
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "=== Migration Validation ==="
|
||||
echo "Repository: $REPO_ROOT"
|
||||
echo ""
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Track issues
|
||||
ERRORS=()
|
||||
WARNINGS=()
|
||||
|
||||
# Function to check for duplicates in a directory
|
||||
check_duplicates() {
|
||||
local dir="$1"
|
||||
local module="$2"
|
||||
|
||||
if [ ! -d "$dir" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Extract numeric prefixes and find duplicates
|
||||
local duplicates
|
||||
duplicates=$(find "$dir" -maxdepth 1 -name "*.sql" -printf "%f\n" 2>/dev/null | \
|
||||
sed -E 's/^([0-9]+)_.*/\1/' | \
|
||||
sort | uniq -d)
|
||||
|
||||
if [ -n "$duplicates" ]; then
|
||||
for prefix in $duplicates; do
|
||||
local files
|
||||
files=$(find "$dir" -maxdepth 1 -name "${prefix}_*.sql" -printf "%f\n" | tr '\n' ', ' | sed 's/,$//')
|
||||
ERRORS+=("[$module] Duplicate prefix $prefix: $files")
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to check naming convention
|
||||
check_naming() {
|
||||
local dir="$1"
|
||||
local module="$2"
|
||||
|
||||
if [ ! -d "$dir" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
find "$dir" -maxdepth 1 -name "*.sql" -printf "%f\n" 2>/dev/null | while read -r file; do
|
||||
# Check standard pattern: NNN_description.sql
|
||||
if [[ "$file" =~ ^[0-9]{3}_[a-z0-9_]+\.sql$ ]]; then
|
||||
continue # Valid standard
|
||||
fi
|
||||
# Check seed pattern: SNNN_description.sql
|
||||
if [[ "$file" =~ ^S[0-9]{3}_[a-z0-9_]+\.sql$ ]]; then
|
||||
continue # Valid seed
|
||||
fi
|
||||
# Check data migration pattern: DMNNN_description.sql
|
||||
if [[ "$file" =~ ^DM[0-9]{3}_[a-z0-9_]+\.sql$ ]]; then
|
||||
continue # Valid data migration
|
||||
fi
|
||||
# Check for Flyway-style
|
||||
if [[ "$file" =~ ^V[0-9]+.*\.sql$ ]]; then
|
||||
WARNINGS+=("[$module] Flyway-style naming: $file (consider NNN_description.sql)")
|
||||
continue
|
||||
fi
|
||||
# Check for EF Core timestamp style
|
||||
if [[ "$file" =~ ^[0-9]{14,}_.*\.sql$ ]]; then
|
||||
WARNINGS+=("[$module] EF Core timestamp naming: $file (consider NNN_description.sql)")
|
||||
continue
|
||||
fi
|
||||
# Check for 4-digit prefix
|
||||
if [[ "$file" =~ ^[0-9]{4}_.*\.sql$ ]]; then
|
||||
WARNINGS+=("[$module] 4-digit prefix: $file (standard is 3-digit NNN_description.sql)")
|
||||
continue
|
||||
fi
|
||||
# Non-standard
|
||||
WARNINGS+=("[$module] Non-standard naming: $file")
|
||||
done
|
||||
}
|
||||
|
||||
# Function to check for dangerous operations in startup migrations
|
||||
check_dangerous_ops() {
|
||||
local dir="$1"
|
||||
local module="$2"
|
||||
|
||||
if [ ! -d "$dir" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
find "$dir" -maxdepth 1 -name "*.sql" -printf "%f\n" 2>/dev/null | while read -r file; do
|
||||
local filepath="$dir/$file"
|
||||
local prefix
|
||||
prefix=$(echo "$file" | sed -E 's/^([0-9]+)_.*/\1/')
|
||||
|
||||
# Only check startup migrations (001-099)
|
||||
if [[ "$prefix" =~ ^0[0-9]{2}$ ]] && [ "$prefix" -lt 100 ]; then
|
||||
# Check for DROP TABLE without IF EXISTS
|
||||
if grep -qE "DROP\s+TABLE\s+(?!IF\s+EXISTS)" "$filepath" 2>/dev/null; then
|
||||
ERRORS+=("[$module] $file: DROP TABLE without IF EXISTS in startup migration")
|
||||
fi
|
||||
|
||||
# Check for DROP COLUMN (breaking change in startup)
|
||||
if grep -qiE "ALTER\s+TABLE.*DROP\s+COLUMN" "$filepath" 2>/dev/null; then
|
||||
ERRORS+=("[$module] $file: DROP COLUMN in startup migration (should be release migration 100+)")
|
||||
fi
|
||||
|
||||
# Check for TRUNCATE
|
||||
if grep -qiE "^\s*TRUNCATE" "$filepath" 2>/dev/null; then
|
||||
ERRORS+=("[$module] $file: TRUNCATE in startup migration")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Scan all module migration directories
|
||||
echo "Scanning migration directories..."
|
||||
echo ""
|
||||
|
||||
# Define module migration paths
|
||||
declare -A MIGRATION_PATHS
|
||||
MIGRATION_PATHS=(
|
||||
["Authority"]="src/Authority/__Libraries/StellaOps.Authority.Storage.Postgres/Migrations"
|
||||
["Concelier"]="src/Concelier/__Libraries/StellaOps.Concelier.Storage.Postgres/Migrations"
|
||||
["Excititor"]="src/Excititor/__Libraries/StellaOps.Excititor.Storage.Postgres/Migrations"
|
||||
["Policy"]="src/Policy/__Libraries/StellaOps.Policy.Storage.Postgres/Migrations"
|
||||
["Scheduler"]="src/Scheduler/__Libraries/StellaOps.Scheduler.Storage.Postgres/Migrations"
|
||||
["Notify"]="src/Notify/__Libraries/StellaOps.Notify.Storage.Postgres/Migrations"
|
||||
["Scanner"]="src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/Migrations"
|
||||
["Scanner.Triage"]="src/Scanner/__Libraries/StellaOps.Scanner.Triage/Migrations"
|
||||
["Attestor"]="src/Attestor/__Libraries/StellaOps.Attestor.Persistence/Migrations"
|
||||
["Signer"]="src/Signer/__Libraries/StellaOps.Signer.KeyManagement/Migrations"
|
||||
["Signals"]="src/Signals/StellaOps.Signals.Storage.Postgres/Migrations"
|
||||
["EvidenceLocker"]="src/EvidenceLocker/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.Infrastructure/Db/Migrations"
|
||||
["ExportCenter"]="src/ExportCenter/StellaOps.ExportCenter/StellaOps.ExportCenter.Infrastructure/Db/Migrations"
|
||||
["IssuerDirectory"]="src/IssuerDirectory/StellaOps.IssuerDirectory/StellaOps.IssuerDirectory.Storage.Postgres/Migrations"
|
||||
["Orchestrator"]="src/Orchestrator/StellaOps.Orchestrator/StellaOps.Orchestrator.Infrastructure/migrations"
|
||||
["TimelineIndexer"]="src/TimelineIndexer/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Infrastructure/Db/Migrations"
|
||||
["BinaryIndex"]="src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/Migrations"
|
||||
["Unknowns"]="src/Unknowns/__Libraries/StellaOps.Unknowns.Storage.Postgres/Migrations"
|
||||
["VexHub"]="src/VexHub/__Libraries/StellaOps.VexHub.Storage.Postgres/Migrations"
|
||||
)
|
||||
|
||||
for module in "${!MIGRATION_PATHS[@]}"; do
|
||||
path="$REPO_ROOT/${MIGRATION_PATHS[$module]}"
|
||||
if [ -d "$path" ]; then
|
||||
echo "Checking: $module"
|
||||
check_duplicates "$path" "$module"
|
||||
check_naming "$path" "$module"
|
||||
check_dangerous_ops "$path" "$module"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Report errors
|
||||
if [ ${#ERRORS[@]} -gt 0 ]; then
|
||||
echo -e "${RED}=== ERRORS (${#ERRORS[@]}) ===${NC}"
|
||||
for error in "${ERRORS[@]}"; do
|
||||
echo -e "${RED} ✗ $error${NC}"
|
||||
done
|
||||
EXIT_CODE=1
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Report warnings
|
||||
if [ ${#WARNINGS[@]} -gt 0 ]; then
|
||||
echo -e "${YELLOW}=== WARNINGS (${#WARNINGS[@]}) ===${NC}"
|
||||
for warning in "${WARNINGS[@]}"; do
|
||||
echo -e "${YELLOW} ⚠ $warning${NC}"
|
||||
done
|
||||
if [ "$STRICT_MODE" = true ]; then
|
||||
EXIT_CODE=1
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Scanner fix suggestions
|
||||
if [ "$FIX_SCANNER" = true ]; then
|
||||
echo "=== Scanner Migration Rename Suggestions ==="
|
||||
echo "# Run these commands to fix Scanner duplicate migrations:"
|
||||
echo ""
|
||||
|
||||
SCANNER_DIR="$REPO_ROOT/src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/Migrations"
|
||||
if [ -d "$SCANNER_DIR" ]; then
|
||||
# Map old names to new sequential numbers
|
||||
cat << 'EOF'
|
||||
# Before running: backup the schema_migrations table!
|
||||
# After renaming: update schema_migrations.migration_name to match new names
|
||||
|
||||
cd src/Scanner/__Libraries/StellaOps.Scanner.Storage/Postgres/Migrations
|
||||
|
||||
# Fix duplicate 009 prefixes
|
||||
git mv 009_call_graph_tables.sql 020_call_graph_tables.sql
|
||||
git mv 009_smart_diff_tables_search_path.sql 021_smart_diff_tables_search_path.sql
|
||||
|
||||
# Fix duplicate 010 prefixes
|
||||
git mv 010_reachability_drift_tables.sql 022_reachability_drift_tables.sql
|
||||
git mv 010_scanner_api_ingestion.sql 023_scanner_api_ingestion.sql
|
||||
git mv 010_smart_diff_priority_score_widen.sql 024_smart_diff_priority_score_widen.sql
|
||||
|
||||
# Fix duplicate 014 prefixes
|
||||
git mv 014_epss_triage_columns.sql 025_epss_triage_columns.sql
|
||||
git mv 014_vuln_surfaces.sql 026_vuln_surfaces.sql
|
||||
|
||||
# Renumber subsequent migrations
|
||||
git mv 011_epss_raw_layer.sql 027_epss_raw_layer.sql
|
||||
git mv 012_epss_signal_layer.sql 028_epss_signal_layer.sql
|
||||
git mv 013_witness_storage.sql 029_witness_storage.sql
|
||||
git mv 015_vuln_surface_triggers_update.sql 030_vuln_surface_triggers_update.sql
|
||||
git mv 016_reach_cache.sql 031_reach_cache.sql
|
||||
git mv 017_idempotency_keys.sql 032_idempotency_keys.sql
|
||||
git mv 018_binary_evidence.sql 033_binary_evidence.sql
|
||||
git mv 019_func_proof_tables.sql 034_func_proof_tables.sql
|
||||
EOF
|
||||
fi
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Summary
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}=== VALIDATION PASSED ===${NC}"
|
||||
else
|
||||
echo -e "${RED}=== VALIDATION FAILED ===${NC}"
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
Reference in New Issue
Block a user