284 lines
10 KiB
YAML
284 lines
10 KiB
YAML
# Sprint: Testing Enhancement Advisory - Phase 2.2/2.3
|
|
# Multi-site federation integration tests
|
|
# Tests 3+ site federation scenarios including partitions and latency
|
|
|
|
name: federation-multisite
|
|
|
|
on:
|
|
schedule:
|
|
# Run nightly at 02:00 UTC
|
|
- cron: '0 2 * * *'
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- 'src/Concelier/__Libraries/StellaOps.Concelier.Federation/**'
|
|
- 'src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/**'
|
|
pull_request:
|
|
branches: [main, develop]
|
|
paths:
|
|
- 'src/Concelier/__Libraries/StellaOps.Concelier.Federation/**'
|
|
- 'src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/**'
|
|
workflow_dispatch:
|
|
inputs:
|
|
run_latency_stress:
|
|
description: 'Run extended latency stress tests'
|
|
type: boolean
|
|
default: false
|
|
run_chaos_scenarios:
|
|
description: 'Run chaos/partition scenarios'
|
|
type: boolean
|
|
default: false
|
|
|
|
concurrency:
|
|
group: federation-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# ==========================================================================
|
|
# Multi-Site Federation Tests
|
|
# ==========================================================================
|
|
federation-multisite-tests:
|
|
name: Multi-Site Federation Tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: "10.0.100"
|
|
|
|
- name: Restore dependencies
|
|
run: dotnet restore src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj
|
|
|
|
- name: Build federation tests
|
|
run: dotnet build src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj --configuration Release --no-restore
|
|
|
|
- name: Run 3-Site Convergence Tests
|
|
run: |
|
|
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
|
|
--filter "Category=Federation&FullyQualifiedName~ThreeSite" \
|
|
--configuration Release \
|
|
--no-build \
|
|
--logger "trx;LogFileName=federation-convergence.trx" \
|
|
--results-directory ./TestResults
|
|
|
|
- name: Run Partition Tests
|
|
run: |
|
|
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
|
|
--filter "Category=Federation&FullyQualifiedName~Partition" \
|
|
--configuration Release \
|
|
--no-build \
|
|
--logger "trx;LogFileName=federation-partition.trx" \
|
|
--results-directory ./TestResults
|
|
|
|
- name: Run Latency Tests
|
|
run: |
|
|
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
|
|
--filter "Category=Latency" \
|
|
--configuration Release \
|
|
--no-build \
|
|
--logger "trx;LogFileName=federation-latency.trx" \
|
|
--results-directory ./TestResults
|
|
|
|
- name: Upload test results
|
|
uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: federation-test-results
|
|
path: TestResults/**/*.trx
|
|
|
|
- name: Publish test summary
|
|
uses: dorny/test-reporter@v1
|
|
if: always()
|
|
with:
|
|
name: Federation Test Results
|
|
path: TestResults/**/*.trx
|
|
reporter: dotnet-trx
|
|
|
|
# ==========================================================================
|
|
# Extended Latency Stress Tests (On-Demand)
|
|
# ==========================================================================
|
|
latency-stress-tests:
|
|
name: Latency Stress Tests
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'workflow_dispatch' && github.event.inputs.run_latency_stress == 'true'
|
|
timeout-minutes: 60
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: "10.0.100"
|
|
|
|
- name: Build federation tests
|
|
run: dotnet build src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj --configuration Release
|
|
|
|
- name: Run Extended Latency Scenarios
|
|
run: |
|
|
# Run cross-region tests with various latency configurations
|
|
for LATENCY in 100 500 1000 2000; do
|
|
echo "Testing with ${LATENCY}ms latency..."
|
|
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
|
|
--filter "Category=Latency&FullyQualifiedName~CrossRegion" \
|
|
--configuration Release \
|
|
--no-build \
|
|
--logger "trx;LogFileName=latency-stress-${LATENCY}ms.trx" \
|
|
--results-directory ./TestResults/latency-stress || true
|
|
done
|
|
|
|
- name: Analyze latency results
|
|
run: |
|
|
echo "Latency stress test results:"
|
|
find ./TestResults -name "*.trx" -exec basename {} \;
|
|
|
|
- name: Upload stress test results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: latency-stress-results
|
|
path: TestResults/**
|
|
|
|
# ==========================================================================
|
|
# Chaos Scenario Tests (On-Demand)
|
|
# ==========================================================================
|
|
chaos-scenario-tests:
|
|
name: Chaos Scenario Tests
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'workflow_dispatch' && github.event.inputs.run_chaos_scenarios == 'true'
|
|
timeout-minutes: 45
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: "10.0.100"
|
|
|
|
- name: Build federation tests
|
|
run: dotnet build src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj --configuration Release
|
|
|
|
- name: Run Split Brain Scenarios
|
|
run: |
|
|
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
|
|
--filter "Category=Chaos&FullyQualifiedName~SplitBrain" \
|
|
--configuration Release \
|
|
--no-build \
|
|
--logger "trx;LogFileName=chaos-splitbrain.trx" \
|
|
--results-directory ./TestResults
|
|
|
|
- name: Run Flapping Network Scenarios
|
|
run: |
|
|
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
|
|
--filter "Category=Chaos&FullyQualifiedName~Flap" \
|
|
--configuration Release \
|
|
--no-build \
|
|
--logger "trx;LogFileName=chaos-flapping.trx" \
|
|
--results-directory ./TestResults
|
|
|
|
- name: Run Partition Healing Scenarios
|
|
run: |
|
|
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
|
|
--filter "Category=Chaos&FullyQualifiedName~Heal" \
|
|
--configuration Release \
|
|
--no-build \
|
|
--logger "trx;LogFileName=chaos-healing.trx" \
|
|
--results-directory ./TestResults
|
|
|
|
- name: Upload chaos test results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: chaos-test-results
|
|
path: TestResults/**
|
|
|
|
# ==========================================================================
|
|
# Nightly Full Federation Suite
|
|
# ==========================================================================
|
|
nightly-full-suite:
|
|
name: Nightly Full Federation Suite
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'schedule'
|
|
timeout-minutes: 90
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: "10.0.100"
|
|
|
|
- name: Build all federation tests
|
|
run: dotnet build src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests/StellaOps.Concelier.Federation.Tests.csproj --configuration Release
|
|
|
|
- name: Run complete federation test suite
|
|
run: |
|
|
dotnet test src/Concelier/__Tests/StellaOps.Concelier.Federation.Tests \
|
|
--configuration Release \
|
|
--no-build \
|
|
--collect:"XPlat Code Coverage" \
|
|
--logger "trx;LogFileName=federation-full.trx" \
|
|
--results-directory ./TestResults
|
|
|
|
- name: Generate test report
|
|
run: |
|
|
echo "# Federation Test Report" > ./TestResults/report.md
|
|
echo "" >> ./TestResults/report.md
|
|
echo "Run date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> ./TestResults/report.md
|
|
echo "" >> ./TestResults/report.md
|
|
echo "## Test Categories" >> ./TestResults/report.md
|
|
echo "- Multi-site convergence" >> ./TestResults/report.md
|
|
echo "- Network partition handling" >> ./TestResults/report.md
|
|
echo "- Cross-region latency" >> ./TestResults/report.md
|
|
echo "- Split-brain recovery" >> ./TestResults/report.md
|
|
|
|
- name: Upload nightly results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: nightly-federation-results
|
|
path: TestResults/**
|
|
|
|
- name: Send notification on failure
|
|
if: failure()
|
|
run: |
|
|
echo "Federation nightly tests failed - notification would be sent here"
|
|
# Could integrate with Slack/Teams/Email notification
|
|
|
|
# ==========================================================================
|
|
# Test Result Summary
|
|
# ==========================================================================
|
|
test-summary:
|
|
name: Test Summary
|
|
runs-on: ubuntu-latest
|
|
needs: [federation-multisite-tests]
|
|
if: always()
|
|
|
|
steps:
|
|
- name: Download test results
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: federation-test-results
|
|
path: ./TestResults
|
|
|
|
- name: Summarize results
|
|
run: |
|
|
echo "## Federation Test Summary" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "Test categories executed:" >> $GITHUB_STEP_SUMMARY
|
|
echo "- Three-site convergence tests" >> $GITHUB_STEP_SUMMARY
|
|
echo "- Partition/split-brain tests" >> $GITHUB_STEP_SUMMARY
|
|
echo "- Cross-region latency tests" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "Result files:" >> $GITHUB_STEP_SUMMARY
|
|
find ./TestResults -name "*.trx" -exec basename {} \; | while read f; do
|
|
echo "- $f" >> $GITHUB_STEP_SUMMARY
|
|
done
|