# .gitea/workflows/nightly-regression.yml # Nightly Full-Suite Regression Testing # Sprint: CI/CD Enhancement - Comprehensive Testing # # Purpose: Run comprehensive regression tests that are too expensive for PR gating # - Full test matrix (all categories) # - Extended integration tests # - Performance benchmarks with historical comparison # - Cross-module dependency validation # - Determinism verification # # Schedule: Daily at 2:00 AM UTC (off-peak hours) # # Notifications: Slack/Teams on failure name: Nightly Regression on: schedule: - cron: '0 2 * * *' # Daily at 2:00 AM UTC workflow_dispatch: inputs: skip_performance: description: 'Skip performance tests' type: boolean default: false skip_determinism: description: 'Skip determinism tests' type: boolean default: false notify_on_success: description: 'Send notification on success' type: boolean default: false env: DOTNET_VERSION: '10.0.100' DOTNET_NOLOGO: 1 DOTNET_CLI_TELEMETRY_OPTOUT: 1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1 TZ: UTC jobs: # =========================================================================== # PREPARE NIGHTLY RUN # =========================================================================== prepare: name: Prepare Nightly Run runs-on: ubuntu-22.04 outputs: run_id: ${{ steps.metadata.outputs.run_id }} run_date: ${{ steps.metadata.outputs.run_date }} commit_sha: ${{ steps.metadata.outputs.commit_sha }} steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - name: Generate run metadata id: metadata run: | RUN_ID="nightly-$(date -u +%Y%m%d-%H%M%S)" RUN_DATE=$(date -u +%Y-%m-%d) COMMIT_SHA=$(git rev-parse HEAD) echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT echo "run_date=$RUN_DATE" >> $GITHUB_OUTPUT echo "commit_sha=$COMMIT_SHA" >> $GITHUB_OUTPUT echo "## Nightly Regression Run" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "- **Run ID:** $RUN_ID" >> $GITHUB_STEP_SUMMARY echo "- **Date:** $RUN_DATE" >> $GITHUB_STEP_SUMMARY echo "- **Commit:** $COMMIT_SHA" >> $GITHUB_STEP_SUMMARY - name: Check recent commits run: | echo "### Recent Commits" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY git log --oneline -10 >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY # =========================================================================== # FULL BUILD VERIFICATION # =========================================================================== build: name: Full Build runs-on: ubuntu-22.04 timeout-minutes: 30 needs: prepare steps: - name: Checkout uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ env.DOTNET_VERSION }} include-prerelease: true - name: Restore dependencies run: dotnet restore src/StellaOps.sln - name: Build solution (Release) run: | START_TIME=$(date +%s) dotnet build src/StellaOps.sln --configuration Release --no-restore END_TIME=$(date +%s) DURATION=$((END_TIME - START_TIME)) echo "build_time=$DURATION" >> $GITHUB_ENV echo "Build completed in ${DURATION}s" - name: Report build metrics run: | echo "### Build Metrics" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "- **Build Time:** ${{ env.build_time }}s" >> $GITHUB_STEP_SUMMARY echo "- **Configuration:** Release" >> $GITHUB_STEP_SUMMARY # =========================================================================== # COMPREHENSIVE TEST SUITE # =========================================================================== test-pr-gating: name: PR-Gating Tests runs-on: ubuntu-22.04 timeout-minutes: 45 needs: build services: postgres: image: postgres:16 env: POSTGRES_USER: stellaops POSTGRES_PASSWORD: stellaops POSTGRES_DB: stellaops_test ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 strategy: fail-fast: false matrix: category: - Unit - Architecture - Contract - Integration - Security - Golden steps: - name: Checkout uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ env.DOTNET_VERSION }} include-prerelease: true - name: Run ${{ matrix.category }} Tests env: STELLAOPS_TEST_POSTGRES_CONNECTION: "Host=localhost;Port=5432;Database=stellaops_test;Username=stellaops;Password=stellaops" run: | chmod +x .gitea/scripts/test/run-test-category.sh .gitea/scripts/test/run-test-category.sh "${{ matrix.category }}" - name: Upload Test Results uses: actions/upload-artifact@v4 if: always() with: name: nightly-test-${{ matrix.category }} path: ./TestResults/${{ matrix.category }} retention-days: 30 test-extended: name: Extended Tests runs-on: ubuntu-22.04 timeout-minutes: 60 needs: build if: github.event.inputs.skip_performance != 'true' strategy: fail-fast: false matrix: category: - Performance - Benchmark - Resilience - Observability steps: - name: Checkout uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ env.DOTNET_VERSION }} include-prerelease: true - name: Run ${{ matrix.category }} Tests run: | chmod +x .gitea/scripts/test/run-test-category.sh .gitea/scripts/test/run-test-category.sh "${{ matrix.category }}" - name: Upload Test Results uses: actions/upload-artifact@v4 if: always() with: name: nightly-extended-${{ matrix.category }} path: ./TestResults/${{ matrix.category }} retention-days: 30 # =========================================================================== # DETERMINISM VERIFICATION # =========================================================================== determinism: name: Determinism Verification runs-on: ubuntu-22.04 timeout-minutes: 45 needs: build if: github.event.inputs.skip_determinism != 'true' steps: - name: Checkout uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ env.DOTNET_VERSION }} include-prerelease: true - name: First build run: | dotnet build src/StellaOps.sln --configuration Release -o ./build-1 find ./build-1 -name "*.dll" -exec sha256sum {} \; | sort > checksums-1.txt - name: Clean and rebuild run: | rm -rf ./build-1 dotnet clean src/StellaOps.sln dotnet build src/StellaOps.sln --configuration Release -o ./build-2 find ./build-2 -name "*.dll" -exec sha256sum {} \; | sort > checksums-2.txt - name: Compare builds id: compare run: | echo "### Determinism Check" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY if diff checksums-1.txt checksums-2.txt > /dev/null; then echo "PASS: Builds are deterministic" >> $GITHUB_STEP_SUMMARY echo "deterministic=true" >> $GITHUB_OUTPUT else echo "FAIL: Builds differ" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "
Differences" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY diff checksums-1.txt checksums-2.txt >> $GITHUB_STEP_SUMMARY || true echo '```' >> $GITHUB_STEP_SUMMARY echo "
" >> $GITHUB_STEP_SUMMARY echo "deterministic=false" >> $GITHUB_OUTPUT exit 1 fi - name: Upload checksums uses: actions/upload-artifact@v4 if: always() with: name: nightly-determinism-checksums path: checksums-*.txt retention-days: 30 # =========================================================================== # CROSS-MODULE VALIDATION # =========================================================================== cross-module: name: Cross-Module Validation runs-on: ubuntu-22.04 timeout-minutes: 30 needs: build steps: - name: Checkout uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ env.DOTNET_VERSION }} include-prerelease: true - name: Check for circular dependencies run: | echo "### Dependency Analysis" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # Build dependency graph echo "Analyzing project dependencies..." for proj in $(find src -name "*.csproj" ! -path "*/bin/*" ! -path "*/obj/*" | head -50); do # Extract ProjectReference entries refs=$(grep -oP 'ProjectReference Include="\K[^"]+' "$proj" 2>/dev/null || true) if [[ -n "$refs" ]]; then basename "$proj" >> deps.txt echo "$refs" | while read ref; do echo " -> $(basename "$ref")" >> deps.txt done fi done if [[ -f deps.txt ]]; then echo "
Project Dependencies (first 50)" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY head -100 deps.txt >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo "
" >> $GITHUB_STEP_SUMMARY fi - name: Validate no deprecated APIs run: | # Check for use of deprecated patterns DEPRECATED_COUNT=$(grep -r "Obsolete" src --include="*.cs" | wc -l || echo "0") echo "- Obsolete attribute usages: $DEPRECATED_COUNT" >> $GITHUB_STEP_SUMMARY # =========================================================================== # CODE COVERAGE REPORT # =========================================================================== coverage: name: Code Coverage runs-on: ubuntu-22.04 timeout-minutes: 45 needs: build services: postgres: image: postgres:16 env: POSTGRES_USER: stellaops POSTGRES_PASSWORD: stellaops POSTGRES_DB: stellaops_test ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - name: Checkout uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ env.DOTNET_VERSION }} include-prerelease: true - name: Run tests with coverage env: STELLAOPS_TEST_POSTGRES_CONNECTION: "Host=localhost;Port=5432;Database=stellaops_test;Username=stellaops;Password=stellaops" run: | dotnet test src/StellaOps.sln \ --configuration Release \ --collect:"XPlat Code Coverage" \ --results-directory ./TestResults/Coverage \ --filter "Category=Unit|Category=Integration" \ --verbosity minimal \ -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura - name: Install ReportGenerator run: dotnet tool install -g dotnet-reportgenerator-globaltool - name: Generate coverage report run: | reportgenerator \ -reports:"./TestResults/Coverage/**/coverage.cobertura.xml" \ -targetdir:"./TestResults/CoverageReport" \ -reporttypes:"Html;MarkdownSummary;Cobertura" \ || true - name: Add coverage to summary run: | echo "### Code Coverage Report" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY if [[ -f "./TestResults/CoverageReport/Summary.md" ]]; then cat "./TestResults/CoverageReport/Summary.md" >> $GITHUB_STEP_SUMMARY else echo "Coverage report generation failed or no coverage data collected." >> $GITHUB_STEP_SUMMARY fi - name: Upload coverage report uses: actions/upload-artifact@v4 if: always() with: name: nightly-coverage-report path: ./TestResults/CoverageReport retention-days: 30 # =========================================================================== # SUMMARY AND NOTIFICATION # =========================================================================== summary: name: Nightly Summary runs-on: ubuntu-22.04 needs: - prepare - build - test-pr-gating - test-extended - determinism - cross-module - coverage if: always() steps: - name: Generate final summary run: | echo "## Nightly Regression Summary" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Run ID:** ${{ needs.prepare.outputs.run_id }}" >> $GITHUB_STEP_SUMMARY echo "**Date:** ${{ needs.prepare.outputs.run_date }}" >> $GITHUB_STEP_SUMMARY echo "**Commit:** ${{ needs.prepare.outputs.commit_sha }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Job Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY echo "| Build | ${{ needs.build.result }} |" >> $GITHUB_STEP_SUMMARY echo "| PR-Gating Tests | ${{ needs.test-pr-gating.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Extended Tests | ${{ needs.test-extended.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Determinism | ${{ needs.determinism.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Cross-Module | ${{ needs.cross-module.result }} |" >> $GITHUB_STEP_SUMMARY echo "| Coverage | ${{ needs.coverage.result }} |" >> $GITHUB_STEP_SUMMARY - name: Determine overall status id: status run: | if [[ "${{ needs.build.result }}" == "failure" ]] || \ [[ "${{ needs.test-pr-gating.result }}" == "failure" ]] || \ [[ "${{ needs.determinism.result }}" == "failure" ]]; then echo "status=failure" >> $GITHUB_OUTPUT else echo "status=success" >> $GITHUB_OUTPUT fi # Placeholder for notifications - configure webhook URL in secrets - name: Send failure notification if: steps.status.outputs.status == 'failure' run: | echo "::warning::Nightly regression failed - notification would be sent here" # Uncomment and configure when webhook is available: # curl -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \ # -H "Content-Type: application/json" \ # -d '{ # "text": "Nightly Regression Failed", # "attachments": [{ # "color": "danger", # "fields": [ # {"title": "Run ID", "value": "${{ needs.prepare.outputs.run_id }}", "short": true}, # {"title": "Commit", "value": "${{ needs.prepare.outputs.commit_sha }}", "short": true} # ] # }] # }' - name: Send success notification if: steps.status.outputs.status == 'success' && github.event.inputs.notify_on_success == 'true' run: | echo "::notice::Nightly regression passed" - name: Exit with appropriate code if: steps.status.outputs.status == 'failure' run: exit 1