# Sprint: Testing Enhancement Advisory - Phase 2.2/2.3
# Multi-site federation integration tests
# Tests 3+ site federation scenarios including partitions and latency

name: federation-multisite

on:
  schedule:
    # Run nightly at 02:00 UTC
    - cron: '0 2 * * *'
  push:
    branches: [main]
    paths:
      - 'src/Concelier/__Libraries/StellaOps.Concelier.Federation/**'
      - 'src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/**'
  pull_request:
    branches: [main, develop]
    paths:
      - 'src/Concelier/__Libraries/StellaOps.Concelier.Federation/**'
      - 'src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/**'
  workflow_dispatch:
    inputs:
      run_latency_stress:
        description: 'Run extended latency stress tests'
        type: boolean
        default: false
      run_chaos_scenarios:
        description: 'Run chaos/partition scenarios'
        type: boolean
        default: false

concurrency:
  group: federation-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # ==========================================================================
  # Multi-Site Federation Tests
  # ==========================================================================
  federation-multisite-tests:
    name: Multi-Site Federation Tests
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: "10.0.100"

      - name: Restore dependencies
        run: dotnet restore src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj

      - name: Build federation tests
        run: dotnet build src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj --configuration Release --no-restore

      - name: Run 3-Site Convergence Tests
        run: |
          dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
            --filter "Category=Federation&FullyQualifiedName~ThreeSite" \
            --configuration Release \
            --no-build \
            --logger "trx;LogFileName=federation-convergence.trx" \
            --results-directory ./TestResults

      - name: Run Partition Tests
        run: |
          dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
            --filter "Category=Federation&FullyQualifiedName~Partition" \
            --configuration Release \
            --no-build \
            --logger "trx;LogFileName=federation-partition.trx" \
            --results-directory ./TestResults

      - name: Run Latency Tests
        run: |
          dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
            --filter "Category=Latency" \
            --configuration Release \
            --no-build \
            --logger "trx;LogFileName=federation-latency.trx" \
            --results-directory ./TestResults

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: federation-test-results
          path: TestResults/**/*.trx

      - name: Publish test summary
        uses: dorny/test-reporter@v1
        if: always()
        with:
          name: Federation Test Results
          path: TestResults/**/*.trx
          reporter: dotnet-trx

  # ==========================================================================
  # Extended Latency Stress Tests (On-Demand)
  # ==========================================================================
  latency-stress-tests:
    name: Latency Stress Tests
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.run_latency_stress == 'true'
    timeout-minutes: 60

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: "10.0.100"

      - name: Build federation tests
        run: dotnet build src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj --configuration Release

      - name: Run Extended Latency Scenarios
        run: |
          # Run cross-region tests with various latency configurations
          for LATENCY in 100 500 1000 2000; do
            echo "Testing with ${LATENCY}ms latency..."
            dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
              --filter "Category=Latency&FullyQualifiedName~CrossRegion" \
              --configuration Release \
              --no-build \
              --logger "trx;LogFileName=latency-stress-${LATENCY}ms.trx" \
              --results-directory ./TestResults/latency-stress || true
          done

      - name: Analyze latency results
        run: |
          echo "Latency stress test results:"
          find ./TestResults -name "*.trx" -exec basename {} \;

      - name: Upload stress test results
        uses: actions/upload-artifact@v4
        with:
          name: latency-stress-results
          path: TestResults/**

  # ==========================================================================
  # Chaos Scenario Tests (On-Demand)
  # ==========================================================================
  chaos-scenario-tests:
    name: Chaos Scenario Tests
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.run_chaos_scenarios == 'true'
    timeout-minutes: 45

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: "10.0.100"

      - name: Build federation tests
        run: dotnet build src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj --configuration Release

      - name: Run Split Brain Scenarios
        run: |
          dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
            --filter "Category=Chaos&FullyQualifiedName~SplitBrain" \
            --configuration Release \
            --no-build \
            --logger "trx;LogFileName=chaos-splitbrain.trx" \
            --results-directory ./TestResults

      - name: Run Flapping Network Scenarios
        run: |
          dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
            --filter "Category=Chaos&FullyQualifiedName~Flap" \
            --configuration Release \
            --no-build \
            --logger "trx;LogFileName=chaos-flapping.trx" \
            --results-directory ./TestResults

      - name: Run Partition Healing Scenarios
        run: |
          dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
            --filter "Category=Chaos&FullyQualifiedName~Heal" \
            --configuration Release \
            --no-build \
            --logger "trx;LogFileName=chaos-healing.trx" \
            --results-directory ./TestResults

      - name: Upload chaos test results
        uses: actions/upload-artifact@v4
        with:
          name: chaos-test-results
          path: TestResults/**

  # ==========================================================================
  # Nightly Full Federation Suite
  # ==========================================================================
  nightly-full-suite:
    name: Nightly Full Federation Suite
    runs-on: ubuntu-latest
    if: github.event_name == 'schedule'
    timeout-minutes: 90

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: "10.0.100"

      - name: Build all federation tests
        run: dotnet build src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj --configuration Release

      - name: Run complete federation test suite
        run: |
          dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
            --configuration Release \
            --no-build \
            --collect:"XPlat Code Coverage" \
            --logger "trx;LogFileName=federation-full.trx" \
            --results-directory ./TestResults

      - name: Generate test report
        run: |
          echo "# Federation Test Report" > ./TestResults/report.md
          echo "" >> ./TestResults/report.md
          echo "Run date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> ./TestResults/report.md
          echo "" >> ./TestResults/report.md
          echo "## Test Categories" >> ./TestResults/report.md
          echo "- Multi-site convergence" >> ./TestResults/report.md
          echo "- Network partition handling" >> ./TestResults/report.md
          echo "- Cross-region latency" >> ./TestResults/report.md
          echo "- Split-brain recovery" >> ./TestResults/report.md

      - name: Upload nightly results
        uses: actions/upload-artifact@v4
        with:
          name: nightly-federation-results
          path: TestResults/**

      - name: Send notification on failure
        if: failure()
        run: |
          echo "Federation nightly tests failed - notification would be sent here"
          # Could integrate with Slack/Teams/Email notification

  # ==========================================================================
  # Test Result Summary
  # ==========================================================================
  test-summary:
    name: Test Summary
    runs-on: ubuntu-latest
    needs: [federation-multisite-tests]
    if: always()

    steps:
      - name: Download test results
        uses: actions/download-artifact@v4
        with:
          name: federation-test-results
          path: ./TestResults

      - name: Summarize results
        run: |
          echo "## Federation Test Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Test categories executed:" >> $GITHUB_STEP_SUMMARY
          echo "- Three-site convergence tests" >> $GITHUB_STEP_SUMMARY
          echo "- Partition/split-brain tests" >> $GITHUB_STEP_SUMMARY
          echo "- Cross-region latency tests" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Result files:" >> $GITHUB_STEP_SUMMARY
          find ./TestResults -name "*.trx" -exec basename {} \; | while read f; do
            echo "- $f" >> $GITHUB_STEP_SUMMARY
          done