Files
git.stella-ops.org/.gitea/workflows/migration-test.yml

513 lines
20 KiB
YAML

# .gitea/workflows/migration-test.yml
# Database Migration Testing Workflow
# Sprint: CI/CD Enhancement - Migration Safety
#
# Purpose: Validate database migrations work correctly in both directions
# - Forward migrations (upgrade)
# - Backward migrations (rollback)
# - Idempotency checks (re-running migrations)
# - Data integrity verification
#
# Triggers:
# - Pull requests that modify migration files
# - Scheduled daily validation
# - Manual dispatch for full migration suite
#
# Prerequisites:
# - PostgreSQL 16+ database
# - EF Core migrations in src/**/Migrations/
# - Migration scripts in devops/database/migrations/
name: Migration Testing
on:
push:
branches: [main]
paths:
- '**/Migrations/**'
- 'devops/database/**'
pull_request:
paths:
- '**/Migrations/**'
- 'devops/database/**'
schedule:
- cron: '30 4 * * *' # Daily at 4:30 AM UTC
workflow_dispatch:
inputs:
test_rollback:
description: 'Test rollback migrations'
type: boolean
default: true
test_idempotency:
description: 'Test migration idempotency'
type: boolean
default: true
target_module:
description: 'Specific module to test (empty = all)'
type: string
default: ''
baseline_version:
description: 'Baseline version to test from'
type: string
default: ''
env:
DOTNET_VERSION: '10.0.100'
DOTNET_NOLOGO: 1
DOTNET_CLI_TELEMETRY_OPTOUT: 1
TZ: UTC
POSTGRES_HOST: localhost
POSTGRES_PORT: 5432
POSTGRES_USER: stellaops_migration
POSTGRES_PASSWORD: migration_test_password
POSTGRES_DB: stellaops_migration_test
jobs:
# ===========================================================================
# DISCOVER MODULES WITH MIGRATIONS
# ===========================================================================
discover:
name: Discover Migrations
runs-on: ubuntu-22.04
outputs:
modules: ${{ steps.find.outputs.modules }}
module_count: ${{ steps.find.outputs.count }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Find modules with migrations
id: find
run: |
# Find all EF Core migration directories
MODULES=$(find src -type d -name "Migrations" -path "*/Persistence/*" | \
sed 's|/Migrations||' | \
sort -u | \
jq -R -s -c 'split("\n") | map(select(length > 0))')
COUNT=$(echo "$MODULES" | jq 'length')
echo "Found $COUNT modules with migrations"
echo "$MODULES" | jq -r '.[]'
# Filter by target module if specified
if [[ -n "${{ github.event.inputs.target_module }}" ]]; then
MODULES=$(echo "$MODULES" | jq -c --arg target "${{ github.event.inputs.target_module }}" \
'map(select(contains($target)))')
COUNT=$(echo "$MODULES" | jq 'length')
echo "Filtered to $COUNT modules matching: ${{ github.event.inputs.target_module }}"
fi
echo "modules=$MODULES" >> $GITHUB_OUTPUT
echo "count=$COUNT" >> $GITHUB_OUTPUT
- name: Display discovered modules
run: |
echo "## Discovered Migration Modules" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Module | Path |" >> $GITHUB_STEP_SUMMARY
echo "|--------|------|" >> $GITHUB_STEP_SUMMARY
for path in $(echo '${{ steps.find.outputs.modules }}' | jq -r '.[]'); do
module=$(basename $(dirname "$path"))
echo "| $module | $path |" >> $GITHUB_STEP_SUMMARY
done
# ===========================================================================
# FORWARD MIGRATION TESTS
# ===========================================================================
forward-migrations:
name: Forward Migration
runs-on: ubuntu-22.04
timeout-minutes: 30
needs: discover
if: needs.discover.outputs.module_count != '0'
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: ${{ env.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
POSTGRES_DB: ${{ env.POSTGRES_DB }}
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
module: ${{ fromJson(needs.discover.outputs.modules) }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: ${{ env.DOTNET_VERSION }}
include-prerelease: true
- name: Install EF Core tools
run: dotnet tool install -g dotnet-ef
- name: Get module name
id: module
run: |
MODULE_NAME=$(basename $(dirname "${{ matrix.module }}"))
echo "name=$MODULE_NAME" >> $GITHUB_OUTPUT
echo "Testing module: $MODULE_NAME"
- name: Find project file
id: project
run: |
# Find the csproj file in the persistence directory
PROJECT_FILE=$(find "${{ matrix.module }}" -maxdepth 1 -name "*.csproj" | head -1)
if [[ -z "$PROJECT_FILE" ]]; then
echo "::error::No project file found in ${{ matrix.module }}"
exit 1
fi
echo "project=$PROJECT_FILE" >> $GITHUB_OUTPUT
echo "Found project: $PROJECT_FILE"
- name: Create fresh database
run: |
PGPASSWORD=${{ env.POSTGRES_PASSWORD }} psql -h ${{ env.POSTGRES_HOST }} \
-U ${{ env.POSTGRES_USER }} -d postgres \
-c "DROP DATABASE IF EXISTS ${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }};"
PGPASSWORD=${{ env.POSTGRES_PASSWORD }} psql -h ${{ env.POSTGRES_HOST }} \
-U ${{ env.POSTGRES_USER }} -d postgres \
-c "CREATE DATABASE ${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }};"
- name: Apply all migrations (forward)
id: forward
env:
ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
run: |
echo "Applying migrations for ${{ steps.module.outputs.name }}..."
# List available migrations first
dotnet ef migrations list --project "${{ steps.project.outputs.project }}" \
--no-build 2>/dev/null || true
# Apply all migrations
START_TIME=$(date +%s)
dotnet ef database update --project "${{ steps.project.outputs.project }}"
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
echo "duration=$DURATION" >> $GITHUB_OUTPUT
echo "Migration completed in ${DURATION}s"
- name: Verify schema
env:
PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
run: |
echo "## Schema verification for ${{ steps.module.outputs.name }}" >> $GITHUB_STEP_SUMMARY
# Get table count
TABLE_COUNT=$(psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} \
-d "${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }}" -t -c \
"SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';")
echo "- Tables created: $TABLE_COUNT" >> $GITHUB_STEP_SUMMARY
echo "- Migration time: ${{ steps.forward.outputs.duration }}s" >> $GITHUB_STEP_SUMMARY
# List tables
echo "" >> $GITHUB_STEP_SUMMARY
echo "<details><summary>Tables</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} \
-d "${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }}" -c \
"SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name;" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: Upload migration log
uses: actions/upload-artifact@v4
if: always()
with:
name: migration-forward-${{ steps.module.outputs.name }}
path: |
**/*.migration.log
retention-days: 7
# ===========================================================================
# ROLLBACK MIGRATION TESTS
# ===========================================================================
rollback-migrations:
name: Rollback Migration
runs-on: ubuntu-22.04
timeout-minutes: 30
needs: [discover, forward-migrations]
if: |
needs.discover.outputs.module_count != '0' &&
(github.event_name == 'schedule' || github.event.inputs.test_rollback == 'true')
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: ${{ env.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
POSTGRES_DB: ${{ env.POSTGRES_DB }}
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
module: ${{ fromJson(needs.discover.outputs.modules) }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: ${{ env.DOTNET_VERSION }}
include-prerelease: true
- name: Install EF Core tools
run: dotnet tool install -g dotnet-ef
- name: Get module info
id: module
run: |
MODULE_NAME=$(basename $(dirname "${{ matrix.module }}"))
echo "name=$MODULE_NAME" >> $GITHUB_OUTPUT
PROJECT_FILE=$(find "${{ matrix.module }}" -maxdepth 1 -name "*.csproj" | head -1)
echo "project=$PROJECT_FILE" >> $GITHUB_OUTPUT
- name: Create and migrate database
env:
ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
run: |
# Create database
psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d postgres \
-c "DROP DATABASE IF EXISTS ${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};"
psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d postgres \
-c "CREATE DATABASE ${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};"
# Apply all migrations
dotnet ef database update --project "${{ steps.module.outputs.project }}"
- name: Get migration list
id: migrations
env:
ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
run: |
# Get list of applied migrations
MIGRATIONS=$(dotnet ef migrations list --project "${{ steps.module.outputs.project }}" \
--no-build 2>/dev/null | grep -E "^\d{14}_" | tail -5)
MIGRATION_COUNT=$(echo "$MIGRATIONS" | wc -l)
echo "count=$MIGRATION_COUNT" >> $GITHUB_OUTPUT
if [[ $MIGRATION_COUNT -gt 1 ]]; then
# Get the second-to-last migration for rollback target
ROLLBACK_TARGET=$(echo "$MIGRATIONS" | tail -2 | head -1)
echo "rollback_to=$ROLLBACK_TARGET" >> $GITHUB_OUTPUT
echo "Will rollback to: $ROLLBACK_TARGET"
else
echo "rollback_to=" >> $GITHUB_OUTPUT
echo "Not enough migrations to test rollback"
fi
- name: Test rollback
if: steps.migrations.outputs.rollback_to != ''
env:
ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
run: |
echo "Rolling back to: ${{ steps.migrations.outputs.rollback_to }}"
dotnet ef database update "${{ steps.migrations.outputs.rollback_to }}" \
--project "${{ steps.module.outputs.project }}"
echo "Rollback successful!"
- name: Test re-apply after rollback
if: steps.migrations.outputs.rollback_to != ''
env:
ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
run: |
echo "Re-applying migrations after rollback..."
dotnet ef database update --project "${{ steps.module.outputs.project }}"
echo "Re-apply successful!"
- name: Report rollback results
if: always()
run: |
echo "## Rollback Test: ${{ steps.module.outputs.name }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [[ -n "${{ steps.migrations.outputs.rollback_to }}" ]]; then
echo "- Rollback target: ${{ steps.migrations.outputs.rollback_to }}" >> $GITHUB_STEP_SUMMARY
echo "- Status: Tested" >> $GITHUB_STEP_SUMMARY
else
echo "- Status: Skipped (insufficient migrations)" >> $GITHUB_STEP_SUMMARY
fi
# ===========================================================================
# IDEMPOTENCY TESTS
# ===========================================================================
idempotency:
name: Idempotency Test
runs-on: ubuntu-22.04
timeout-minutes: 20
needs: [discover, forward-migrations]
if: |
needs.discover.outputs.module_count != '0' &&
(github.event_name == 'schedule' || github.event.inputs.test_idempotency == 'true')
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: ${{ env.POSTGRES_USER }}
POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
POSTGRES_DB: ${{ env.POSTGRES_DB }}
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
strategy:
fail-fast: false
matrix:
module: ${{ fromJson(needs.discover.outputs.modules) }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: ${{ env.DOTNET_VERSION }}
include-prerelease: true
- name: Install EF Core tools
run: dotnet tool install -g dotnet-ef
- name: Get module info
id: module
run: |
MODULE_NAME=$(basename $(dirname "${{ matrix.module }}"))
echo "name=$MODULE_NAME" >> $GITHUB_OUTPUT
PROJECT_FILE=$(find "${{ matrix.module }}" -maxdepth 1 -name "*.csproj" | head -1)
echo "project=$PROJECT_FILE" >> $GITHUB_OUTPUT
- name: Setup database
env:
ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
run: |
psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d postgres \
-c "DROP DATABASE IF EXISTS ${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};"
psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d postgres \
-c "CREATE DATABASE ${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};"
- name: First migration run
env:
ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
run: |
dotnet ef database update --project "${{ steps.module.outputs.project }}"
- name: Get initial schema hash
id: hash1
env:
PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
run: |
SCHEMA_HASH=$(psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} \
-d "${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }}" -t -c \
"SELECT md5(string_agg(table_name || column_name || data_type, '' ORDER BY table_name, column_name))
FROM information_schema.columns WHERE table_schema = 'public';")
echo "hash=$SCHEMA_HASH" >> $GITHUB_OUTPUT
echo "Initial schema hash: $SCHEMA_HASH"
- name: Second migration run (idempotency test)
env:
ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
run: |
# Running migrations again should be a no-op
dotnet ef database update --project "${{ steps.module.outputs.project }}"
- name: Get final schema hash
id: hash2
env:
PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
run: |
SCHEMA_HASH=$(psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} \
-d "${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }}" -t -c \
"SELECT md5(string_agg(table_name || column_name || data_type, '' ORDER BY table_name, column_name))
FROM information_schema.columns WHERE table_schema = 'public';")
echo "hash=$SCHEMA_HASH" >> $GITHUB_OUTPUT
echo "Final schema hash: $SCHEMA_HASH"
- name: Verify idempotency
run: |
HASH1="${{ steps.hash1.outputs.hash }}"
HASH2="${{ steps.hash2.outputs.hash }}"
echo "## Idempotency Test: ${{ steps.module.outputs.name }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "- Initial schema hash: $HASH1" >> $GITHUB_STEP_SUMMARY
echo "- Final schema hash: $HASH2" >> $GITHUB_STEP_SUMMARY
if [[ "$HASH1" == "$HASH2" ]]; then
echo "- Result: PASS (schemas identical)" >> $GITHUB_STEP_SUMMARY
else
echo "- Result: FAIL (schemas differ)" >> $GITHUB_STEP_SUMMARY
echo "::error::Idempotency test failed for ${{ steps.module.outputs.name }}"
exit 1
fi
# ===========================================================================
# SUMMARY
# ===========================================================================
summary:
name: Migration Summary
runs-on: ubuntu-22.04
needs: [discover, forward-migrations, rollback-migrations, idempotency]
if: always()
steps:
- name: Generate Summary
run: |
echo "## Migration Test Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Test | Status |" >> $GITHUB_STEP_SUMMARY
echo "|------|--------|" >> $GITHUB_STEP_SUMMARY
echo "| Discovery | ${{ needs.discover.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Forward Migrations | ${{ needs.forward-migrations.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Rollback Migrations | ${{ needs.rollback-migrations.result }} |" >> $GITHUB_STEP_SUMMARY
echo "| Idempotency | ${{ needs.idempotency.result }} |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Modules Tested: ${{ needs.discover.outputs.module_count }}" >> $GITHUB_STEP_SUMMARY
- name: Check for failures
if: contains(needs.*.result, 'failure')
run: exit 1