save progress

.gitea/workflows/migration-test.yml

@@ -0,0 +1,512 @@
+# .gitea/workflows/migration-test.yml
+# Database Migration Testing Workflow
+# Sprint: CI/CD Enhancement - Migration Safety
+#
+# Purpose: Validate database migrations work correctly in both directions
+#   - Forward migrations (upgrade)
+#   - Backward migrations (rollback)
+#   - Idempotency checks (re-running migrations)
+#   - Data integrity verification
+#
+# Triggers:
+#   - Pull requests that modify migration files
+#   - Scheduled daily validation
+#   - Manual dispatch for full migration suite
+#
+# Prerequisites:
+#   - PostgreSQL 16+ database
+#   - EF Core migrations in src/**/Migrations/
+#   - Migration scripts in devops/database/migrations/
+
+name: Migration Testing
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - '**/Migrations/**'
+      - 'devops/database/**'
+  pull_request:
+    paths:
+      - '**/Migrations/**'
+      - 'devops/database/**'
+  schedule:
+    - cron: '30 4 * * *'  # Daily at 4:30 AM UTC
+  workflow_dispatch:
+    inputs:
+      test_rollback:
+        description: 'Test rollback migrations'
+        type: boolean
+        default: true
+      test_idempotency:
+        description: 'Test migration idempotency'
+        type: boolean
+        default: true
+      target_module:
+        description: 'Specific module to test (empty = all)'
+        type: string
+        default: ''
+      baseline_version:
+        description: 'Baseline version to test from'
+        type: string
+        default: ''
+
+env:
+  DOTNET_VERSION: '10.0.100'
+  DOTNET_NOLOGO: 1
+  DOTNET_CLI_TELEMETRY_OPTOUT: 1
+  TZ: UTC
+  POSTGRES_HOST: localhost
+  POSTGRES_PORT: 5432
+  POSTGRES_USER: stellaops_migration
+  POSTGRES_PASSWORD: migration_test_password
+  POSTGRES_DB: stellaops_migration_test
+
+jobs:
+  # ===========================================================================
+  # DISCOVER MODULES WITH MIGRATIONS
+  # ===========================================================================
+
+  discover:
+    name: Discover Migrations
+    runs-on: ubuntu-22.04
+    outputs:
+      modules: ${{ steps.find.outputs.modules }}
+      module_count: ${{ steps.find.outputs.count }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Find modules with migrations
+        id: find
+        run: |
+          # Find all EF Core migration directories
+          MODULES=$(find src -type d -name "Migrations" -path "*/Persistence/*" | \
+            sed 's|/Migrations||' | \
+            sort -u | \
+            jq -R -s -c 'split("\n") | map(select(length > 0))')
+
+          COUNT=$(echo "$MODULES" | jq 'length')
+
+          echo "Found $COUNT modules with migrations"
+          echo "$MODULES" | jq -r '.[]'
+
+          # Filter by target module if specified
+          if [[ -n "${{ github.event.inputs.target_module }}" ]]; then
+            MODULES=$(echo "$MODULES" | jq -c --arg target "${{ github.event.inputs.target_module }}" \
+              'map(select(contains($target)))')
+            COUNT=$(echo "$MODULES" | jq 'length')
+            echo "Filtered to $COUNT modules matching: ${{ github.event.inputs.target_module }}"
+          fi
+
+          echo "modules=$MODULES" >> $GITHUB_OUTPUT
+          echo "count=$COUNT" >> $GITHUB_OUTPUT
+
+      - name: Display discovered modules
+        run: |
+          echo "## Discovered Migration Modules" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Module | Path |" >> $GITHUB_STEP_SUMMARY
+          echo "|--------|------|" >> $GITHUB_STEP_SUMMARY
+          for path in $(echo '${{ steps.find.outputs.modules }}' | jq -r '.[]'); do
+            module=$(basename $(dirname "$path"))
+            echo "| $module | $path |" >> $GITHUB_STEP_SUMMARY
+          done
+
+  # ===========================================================================
+  # FORWARD MIGRATION TESTS
+  # ===========================================================================
+
+  forward-migrations:
+    name: Forward Migration
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    needs: discover
+    if: needs.discover.outputs.module_count != '0'
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_DB }}
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ${{ fromJson(needs.discover.outputs.modules) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: ${{ env.DOTNET_VERSION }}
+          include-prerelease: true
+
+      - name: Install EF Core tools
+        run: dotnet tool install -g dotnet-ef
+
+      - name: Get module name
+        id: module
+        run: |
+          MODULE_NAME=$(basename $(dirname "${{ matrix.module }}"))
+          echo "name=$MODULE_NAME" >> $GITHUB_OUTPUT
+          echo "Testing module: $MODULE_NAME"
+
+      - name: Find project file
+        id: project
+        run: |
+          # Find the csproj file in the persistence directory
+          PROJECT_FILE=$(find "${{ matrix.module }}" -maxdepth 1 -name "*.csproj" | head -1)
+          if [[ -z "$PROJECT_FILE" ]]; then
+            echo "::error::No project file found in ${{ matrix.module }}"
+            exit 1
+          fi
+          echo "project=$PROJECT_FILE" >> $GITHUB_OUTPUT
+          echo "Found project: $PROJECT_FILE"
+
+      - name: Create fresh database
+        run: |
+          PGPASSWORD=${{ env.POSTGRES_PASSWORD }} psql -h ${{ env.POSTGRES_HOST }} \
+            -U ${{ env.POSTGRES_USER }} -d postgres \
+            -c "DROP DATABASE IF EXISTS ${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }};"
+          PGPASSWORD=${{ env.POSTGRES_PASSWORD }} psql -h ${{ env.POSTGRES_HOST }} \
+            -U ${{ env.POSTGRES_USER }} -d postgres \
+            -c "CREATE DATABASE ${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }};"
+
+      - name: Apply all migrations (forward)
+        id: forward
+        env:
+          ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
+        run: |
+          echo "Applying migrations for ${{ steps.module.outputs.name }}..."
+
+          # List available migrations first
+          dotnet ef migrations list --project "${{ steps.project.outputs.project }}" \
+            --no-build 2>/dev/null || true
+
+          # Apply all migrations
+          START_TIME=$(date +%s)
+          dotnet ef database update --project "${{ steps.project.outputs.project }}"
+          END_TIME=$(date +%s)
+          DURATION=$((END_TIME - START_TIME))
+
+          echo "duration=$DURATION" >> $GITHUB_OUTPUT
+          echo "Migration completed in ${DURATION}s"
+
+      - name: Verify schema
+        env:
+          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
+        run: |
+          echo "## Schema verification for ${{ steps.module.outputs.name }}" >> $GITHUB_STEP_SUMMARY
+
+          # Get table count
+          TABLE_COUNT=$(psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} \
+            -d "${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }}" -t -c \
+            "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';")
+
+          echo "- Tables created: $TABLE_COUNT" >> $GITHUB_STEP_SUMMARY
+          echo "- Migration time: ${{ steps.forward.outputs.duration }}s" >> $GITHUB_STEP_SUMMARY
+
+          # List tables
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "<details><summary>Tables</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} \
+            -d "${{ env.POSTGRES_DB }}_${{ steps.module.outputs.name }}" -c \
+            "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name;" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload migration log
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: migration-forward-${{ steps.module.outputs.name }}
+          path: |
+            **/*.migration.log
+          retention-days: 7
+
+  # ===========================================================================
+  # ROLLBACK MIGRATION TESTS
+  # ===========================================================================
+
+  rollback-migrations:
+    name: Rollback Migration
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    needs: [discover, forward-migrations]
+    if: |
+      needs.discover.outputs.module_count != '0' &&
+      (github.event_name == 'schedule' || github.event.inputs.test_rollback == 'true')
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_DB }}
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ${{ fromJson(needs.discover.outputs.modules) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: ${{ env.DOTNET_VERSION }}
+          include-prerelease: true
+
+      - name: Install EF Core tools
+        run: dotnet tool install -g dotnet-ef
+
+      - name: Get module info
+        id: module
+        run: |
+          MODULE_NAME=$(basename $(dirname "${{ matrix.module }}"))
+          echo "name=$MODULE_NAME" >> $GITHUB_OUTPUT
+
+          PROJECT_FILE=$(find "${{ matrix.module }}" -maxdepth 1 -name "*.csproj" | head -1)
+          echo "project=$PROJECT_FILE" >> $GITHUB_OUTPUT
+
+      - name: Create and migrate database
+        env:
+          ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
+          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
+        run: |
+          # Create database
+          psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d postgres \
+            -c "DROP DATABASE IF EXISTS ${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};"
+          psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d postgres \
+            -c "CREATE DATABASE ${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};"
+
+          # Apply all migrations
+          dotnet ef database update --project "${{ steps.module.outputs.project }}"
+
+      - name: Get migration list
+        id: migrations
+        env:
+          ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
+        run: |
+          # Get list of applied migrations
+          MIGRATIONS=$(dotnet ef migrations list --project "${{ steps.module.outputs.project }}" \
+            --no-build 2>/dev/null | grep -E "^\d{14}_" | tail -5)
+
+          MIGRATION_COUNT=$(echo "$MIGRATIONS" | wc -l)
+          echo "count=$MIGRATION_COUNT" >> $GITHUB_OUTPUT
+
+          if [[ $MIGRATION_COUNT -gt 1 ]]; then
+            # Get the second-to-last migration for rollback target
+            ROLLBACK_TARGET=$(echo "$MIGRATIONS" | tail -2 | head -1)
+            echo "rollback_to=$ROLLBACK_TARGET" >> $GITHUB_OUTPUT
+            echo "Will rollback to: $ROLLBACK_TARGET"
+          else
+            echo "rollback_to=" >> $GITHUB_OUTPUT
+            echo "Not enough migrations to test rollback"
+          fi
+
+      - name: Test rollback
+        if: steps.migrations.outputs.rollback_to != ''
+        env:
+          ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
+        run: |
+          echo "Rolling back to: ${{ steps.migrations.outputs.rollback_to }}"
+          dotnet ef database update "${{ steps.migrations.outputs.rollback_to }}" \
+            --project "${{ steps.module.outputs.project }}"
+
+          echo "Rollback successful!"
+
+      - name: Test re-apply after rollback
+        if: steps.migrations.outputs.rollback_to != ''
+        env:
+          ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_rb_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
+        run: |
+          echo "Re-applying migrations after rollback..."
+          dotnet ef database update --project "${{ steps.module.outputs.project }}"
+
+          echo "Re-apply successful!"
+
+      - name: Report rollback results
+        if: always()
+        run: |
+          echo "## Rollback Test: ${{ steps.module.outputs.name }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          if [[ -n "${{ steps.migrations.outputs.rollback_to }}" ]]; then
+            echo "- Rollback target: ${{ steps.migrations.outputs.rollback_to }}" >> $GITHUB_STEP_SUMMARY
+            echo "- Status: Tested" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "- Status: Skipped (insufficient migrations)" >> $GITHUB_STEP_SUMMARY
+          fi
+
+  # ===========================================================================
+  # IDEMPOTENCY TESTS
+  # ===========================================================================
+
+  idempotency:
+    name: Idempotency Test
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    needs: [discover, forward-migrations]
+    if: |
+      needs.discover.outputs.module_count != '0' &&
+      (github.event_name == 'schedule' || github.event.inputs.test_idempotency == 'true')
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: ${{ env.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
+          POSTGRES_DB: ${{ env.POSTGRES_DB }}
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ${{ fromJson(needs.discover.outputs.modules) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: ${{ env.DOTNET_VERSION }}
+          include-prerelease: true
+
+      - name: Install EF Core tools
+        run: dotnet tool install -g dotnet-ef
+
+      - name: Get module info
+        id: module
+        run: |
+          MODULE_NAME=$(basename $(dirname "${{ matrix.module }}"))
+          echo "name=$MODULE_NAME" >> $GITHUB_OUTPUT
+
+          PROJECT_FILE=$(find "${{ matrix.module }}" -maxdepth 1 -name "*.csproj" | head -1)
+          echo "project=$PROJECT_FILE" >> $GITHUB_OUTPUT
+
+      - name: Setup database
+        env:
+          ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
+          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
+        run: |
+          psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d postgres \
+            -c "DROP DATABASE IF EXISTS ${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};"
+          psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} -d postgres \
+            -c "CREATE DATABASE ${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};"
+
+      - name: First migration run
+        env:
+          ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
+        run: |
+          dotnet ef database update --project "${{ steps.module.outputs.project }}"
+
+      - name: Get initial schema hash
+        id: hash1
+        env:
+          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
+        run: |
+          SCHEMA_HASH=$(psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} \
+            -d "${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }}" -t -c \
+            "SELECT md5(string_agg(table_name || column_name || data_type, '' ORDER BY table_name, column_name))
+             FROM information_schema.columns WHERE table_schema = 'public';")
+          echo "hash=$SCHEMA_HASH" >> $GITHUB_OUTPUT
+          echo "Initial schema hash: $SCHEMA_HASH"
+
+      - name: Second migration run (idempotency test)
+        env:
+          ConnectionStrings__Default: "Host=${{ env.POSTGRES_HOST }};Port=${{ env.POSTGRES_PORT }};Database=${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }};Username=${{ env.POSTGRES_USER }};Password=${{ env.POSTGRES_PASSWORD }}"
+        run: |
+          # Running migrations again should be a no-op
+          dotnet ef database update --project "${{ steps.module.outputs.project }}"
+
+      - name: Get final schema hash
+        id: hash2
+        env:
+          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
+        run: |
+          SCHEMA_HASH=$(psql -h ${{ env.POSTGRES_HOST }} -U ${{ env.POSTGRES_USER }} \
+            -d "${{ env.POSTGRES_DB }}_idem_${{ steps.module.outputs.name }}" -t -c \
+            "SELECT md5(string_agg(table_name || column_name || data_type, '' ORDER BY table_name, column_name))
+             FROM information_schema.columns WHERE table_schema = 'public';")
+          echo "hash=$SCHEMA_HASH" >> $GITHUB_OUTPUT
+          echo "Final schema hash: $SCHEMA_HASH"
+
+      - name: Verify idempotency
+        run: |
+          HASH1="${{ steps.hash1.outputs.hash }}"
+          HASH2="${{ steps.hash2.outputs.hash }}"
+
+          echo "## Idempotency Test: ${{ steps.module.outputs.name }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "- Initial schema hash: $HASH1" >> $GITHUB_STEP_SUMMARY
+          echo "- Final schema hash: $HASH2" >> $GITHUB_STEP_SUMMARY
+
+          if [[ "$HASH1" == "$HASH2" ]]; then
+            echo "- Result: PASS (schemas identical)" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "- Result: FAIL (schemas differ)" >> $GITHUB_STEP_SUMMARY
+            echo "::error::Idempotency test failed for ${{ steps.module.outputs.name }}"
+            exit 1
+          fi
+
+  # ===========================================================================
+  # SUMMARY
+  # ===========================================================================
+
+  summary:
+    name: Migration Summary
+    runs-on: ubuntu-22.04
+    needs: [discover, forward-migrations, rollback-migrations, idempotency]
+    if: always()
+    steps:
+      - name: Generate Summary
+        run: |
+          echo "## Migration Test Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Test | Status |" >> $GITHUB_STEP_SUMMARY
+          echo "|------|--------|" >> $GITHUB_STEP_SUMMARY
+          echo "| Discovery | ${{ needs.discover.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| Forward Migrations | ${{ needs.forward-migrations.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| Rollback Migrations | ${{ needs.rollback-migrations.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| Idempotency | ${{ needs.idempotency.result }} |" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Modules Tested: ${{ needs.discover.outputs.module_count }}" >> $GITHUB_STEP_SUMMARY
+
+      - name: Check for failures
+        if: contains(needs.*.result, 'failure')
+        run: exit 1