484 lines
16 KiB
YAML
484 lines
16 KiB
YAML
# .gitea/workflows/nightly-regression.yml
|
|
# Nightly Full-Suite Regression Testing
|
|
# Sprint: CI/CD Enhancement - Comprehensive Testing
|
|
#
|
|
# Purpose: Run comprehensive regression tests that are too expensive for PR gating
|
|
# - Full test matrix (all categories)
|
|
# - Extended integration tests
|
|
# - Performance benchmarks with historical comparison
|
|
# - Cross-module dependency validation
|
|
# - Determinism verification
|
|
#
|
|
# Schedule: Daily at 2:00 AM UTC (off-peak hours)
|
|
#
|
|
# Notifications: Slack/Teams on failure
|
|
|
|
name: Nightly Regression
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 2 * * *' # Daily at 2:00 AM UTC
|
|
workflow_dispatch:
|
|
inputs:
|
|
skip_performance:
|
|
description: 'Skip performance tests'
|
|
type: boolean
|
|
default: false
|
|
skip_determinism:
|
|
description: 'Skip determinism tests'
|
|
type: boolean
|
|
default: false
|
|
notify_on_success:
|
|
description: 'Send notification on success'
|
|
type: boolean
|
|
default: false
|
|
|
|
env:
|
|
DOTNET_VERSION: '10.0.100'
|
|
DOTNET_NOLOGO: 1
|
|
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
|
DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1
|
|
TZ: UTC
|
|
|
|
jobs:
|
|
# ===========================================================================
|
|
# PREPARE NIGHTLY RUN
|
|
# ===========================================================================
|
|
|
|
prepare:
|
|
name: Prepare Nightly Run
|
|
runs-on: ubuntu-22.04
|
|
outputs:
|
|
run_id: ${{ steps.metadata.outputs.run_id }}
|
|
run_date: ${{ steps.metadata.outputs.run_date }}
|
|
commit_sha: ${{ steps.metadata.outputs.commit_sha }}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Generate run metadata
|
|
id: metadata
|
|
run: |
|
|
RUN_ID="nightly-$(date -u +%Y%m%d-%H%M%S)"
|
|
RUN_DATE=$(date -u +%Y-%m-%d)
|
|
COMMIT_SHA=$(git rev-parse HEAD)
|
|
|
|
echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT
|
|
echo "run_date=$RUN_DATE" >> $GITHUB_OUTPUT
|
|
echo "commit_sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
|
|
|
|
echo "## Nightly Regression Run" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Run ID:** $RUN_ID" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Date:** $RUN_DATE" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Commit:** $COMMIT_SHA" >> $GITHUB_STEP_SUMMARY
|
|
|
|
- name: Check recent commits
|
|
run: |
|
|
echo "### Recent Commits" >> $GITHUB_STEP_SUMMARY
|
|
echo '```' >> $GITHUB_STEP_SUMMARY
|
|
git log --oneline -10 >> $GITHUB_STEP_SUMMARY
|
|
echo '```' >> $GITHUB_STEP_SUMMARY
|
|
|
|
# ===========================================================================
|
|
# FULL BUILD VERIFICATION
|
|
# ===========================================================================
|
|
|
|
build:
|
|
name: Full Build
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
needs: prepare
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
include-prerelease: true
|
|
|
|
- name: Restore dependencies
|
|
run: dotnet restore src/StellaOps.sln
|
|
|
|
- name: Build solution (Release)
|
|
run: |
|
|
START_TIME=$(date +%s)
|
|
dotnet build src/StellaOps.sln --configuration Release --no-restore
|
|
END_TIME=$(date +%s)
|
|
DURATION=$((END_TIME - START_TIME))
|
|
echo "build_time=$DURATION" >> $GITHUB_ENV
|
|
echo "Build completed in ${DURATION}s"
|
|
|
|
- name: Report build metrics
|
|
run: |
|
|
echo "### Build Metrics" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Build Time:** ${{ env.build_time }}s" >> $GITHUB_STEP_SUMMARY
|
|
echo "- **Configuration:** Release" >> $GITHUB_STEP_SUMMARY
|
|
|
|
# ===========================================================================
|
|
# COMPREHENSIVE TEST SUITE
|
|
# ===========================================================================
|
|
|
|
test-pr-gating:
|
|
name: PR-Gating Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 45
|
|
needs: build
|
|
services:
|
|
postgres:
|
|
image: postgres:16
|
|
env:
|
|
POSTGRES_USER: stellaops
|
|
POSTGRES_PASSWORD: stellaops
|
|
POSTGRES_DB: stellaops_test
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
category:
|
|
- Unit
|
|
- Architecture
|
|
- Contract
|
|
- Integration
|
|
- Security
|
|
- Golden
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
include-prerelease: true
|
|
|
|
- name: Run ${{ matrix.category }} Tests
|
|
env:
|
|
STELLAOPS_TEST_POSTGRES_CONNECTION: "Host=localhost;Port=5432;Database=stellaops_test;Username=stellaops;Password=stellaops"
|
|
run: |
|
|
chmod +x .gitea/scripts/test/run-test-category.sh
|
|
.gitea/scripts/test/run-test-category.sh "${{ matrix.category }}"
|
|
|
|
- name: Upload Test Results
|
|
uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: nightly-test-${{ matrix.category }}
|
|
path: ./TestResults/${{ matrix.category }}
|
|
retention-days: 30
|
|
|
|
test-extended:
|
|
name: Extended Tests
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 60
|
|
needs: build
|
|
if: github.event.inputs.skip_performance != 'true'
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
category:
|
|
- Performance
|
|
- Benchmark
|
|
- Resilience
|
|
- Observability
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
include-prerelease: true
|
|
|
|
- name: Run ${{ matrix.category }} Tests
|
|
run: |
|
|
chmod +x .gitea/scripts/test/run-test-category.sh
|
|
.gitea/scripts/test/run-test-category.sh "${{ matrix.category }}"
|
|
|
|
- name: Upload Test Results
|
|
uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: nightly-extended-${{ matrix.category }}
|
|
path: ./TestResults/${{ matrix.category }}
|
|
retention-days: 30
|
|
|
|
# ===========================================================================
|
|
# DETERMINISM VERIFICATION
|
|
# ===========================================================================
|
|
|
|
determinism:
|
|
name: Determinism Verification
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 45
|
|
needs: build
|
|
if: github.event.inputs.skip_determinism != 'true'
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
include-prerelease: true
|
|
|
|
- name: First build
|
|
run: |
|
|
dotnet build src/StellaOps.sln --configuration Release -o ./build-1
|
|
find ./build-1 -name "*.dll" -exec sha256sum {} \; | sort > checksums-1.txt
|
|
|
|
- name: Clean and rebuild
|
|
run: |
|
|
rm -rf ./build-1
|
|
dotnet clean src/StellaOps.sln
|
|
dotnet build src/StellaOps.sln --configuration Release -o ./build-2
|
|
find ./build-2 -name "*.dll" -exec sha256sum {} \; | sort > checksums-2.txt
|
|
|
|
- name: Compare builds
|
|
id: compare
|
|
run: |
|
|
echo "### Determinism Check" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if diff checksums-1.txt checksums-2.txt > /dev/null; then
|
|
echo "PASS: Builds are deterministic" >> $GITHUB_STEP_SUMMARY
|
|
echo "deterministic=true" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "FAIL: Builds differ" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "<details><summary>Differences</summary>" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo '```diff' >> $GITHUB_STEP_SUMMARY
|
|
diff checksums-1.txt checksums-2.txt >> $GITHUB_STEP_SUMMARY || true
|
|
echo '```' >> $GITHUB_STEP_SUMMARY
|
|
echo "</details>" >> $GITHUB_STEP_SUMMARY
|
|
echo "deterministic=false" >> $GITHUB_OUTPUT
|
|
exit 1
|
|
fi
|
|
|
|
- name: Upload checksums
|
|
uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: nightly-determinism-checksums
|
|
path: checksums-*.txt
|
|
retention-days: 30
|
|
|
|
# ===========================================================================
|
|
# CROSS-MODULE VALIDATION
|
|
# ===========================================================================
|
|
|
|
cross-module:
|
|
name: Cross-Module Validation
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
needs: build
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
include-prerelease: true
|
|
|
|
- name: Check for circular dependencies
|
|
run: |
|
|
echo "### Dependency Analysis" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
# Build dependency graph
|
|
echo "Analyzing project dependencies..."
|
|
for proj in $(find src -name "*.csproj" ! -path "*/bin/*" ! -path "*/obj/*" | head -50); do
|
|
# Extract ProjectReference entries
|
|
refs=$(grep -oP 'ProjectReference Include="\K[^"]+' "$proj" 2>/dev/null || true)
|
|
if [[ -n "$refs" ]]; then
|
|
basename "$proj" >> deps.txt
|
|
echo "$refs" | while read ref; do
|
|
echo " -> $(basename "$ref")" >> deps.txt
|
|
done
|
|
fi
|
|
done
|
|
|
|
if [[ -f deps.txt ]]; then
|
|
echo "<details><summary>Project Dependencies (first 50)</summary>" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo '```' >> $GITHUB_STEP_SUMMARY
|
|
head -100 deps.txt >> $GITHUB_STEP_SUMMARY
|
|
echo '```' >> $GITHUB_STEP_SUMMARY
|
|
echo "</details>" >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
|
|
- name: Validate no deprecated APIs
|
|
run: |
|
|
# Check for use of deprecated patterns
|
|
DEPRECATED_COUNT=$(grep -r "Obsolete" src --include="*.cs" | wc -l || echo "0")
|
|
echo "- Obsolete attribute usages: $DEPRECATED_COUNT" >> $GITHUB_STEP_SUMMARY
|
|
|
|
# ===========================================================================
|
|
# CODE COVERAGE REPORT
|
|
# ===========================================================================
|
|
|
|
coverage:
|
|
name: Code Coverage
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 45
|
|
needs: build
|
|
services:
|
|
postgres:
|
|
image: postgres:16
|
|
env:
|
|
POSTGRES_USER: stellaops
|
|
POSTGRES_PASSWORD: stellaops
|
|
POSTGRES_DB: stellaops_test
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
include-prerelease: true
|
|
|
|
- name: Run tests with coverage
|
|
env:
|
|
STELLAOPS_TEST_POSTGRES_CONNECTION: "Host=localhost;Port=5432;Database=stellaops_test;Username=stellaops;Password=stellaops"
|
|
run: |
|
|
dotnet test src/StellaOps.sln \
|
|
--configuration Release \
|
|
--collect:"XPlat Code Coverage" \
|
|
--results-directory ./TestResults/Coverage \
|
|
--filter "Category=Unit|Category=Integration" \
|
|
--verbosity minimal \
|
|
-- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.Format=cobertura
|
|
|
|
- name: Install ReportGenerator
|
|
run: dotnet tool install -g dotnet-reportgenerator-globaltool
|
|
|
|
- name: Generate coverage report
|
|
run: |
|
|
reportgenerator \
|
|
-reports:"./TestResults/Coverage/**/coverage.cobertura.xml" \
|
|
-targetdir:"./TestResults/CoverageReport" \
|
|
-reporttypes:"Html;MarkdownSummary;Cobertura" \
|
|
|| true
|
|
|
|
- name: Add coverage to summary
|
|
run: |
|
|
echo "### Code Coverage Report" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
if [[ -f "./TestResults/CoverageReport/Summary.md" ]]; then
|
|
cat "./TestResults/CoverageReport/Summary.md" >> $GITHUB_STEP_SUMMARY
|
|
else
|
|
echo "Coverage report generation failed or no coverage data collected." >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
|
|
- name: Upload coverage report
|
|
uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: nightly-coverage-report
|
|
path: ./TestResults/CoverageReport
|
|
retention-days: 30
|
|
|
|
# ===========================================================================
|
|
# SUMMARY AND NOTIFICATION
|
|
# ===========================================================================
|
|
|
|
summary:
|
|
name: Nightly Summary
|
|
runs-on: ubuntu-22.04
|
|
needs:
|
|
- prepare
|
|
- build
|
|
- test-pr-gating
|
|
- test-extended
|
|
- determinism
|
|
- cross-module
|
|
- coverage
|
|
if: always()
|
|
steps:
|
|
- name: Generate final summary
|
|
run: |
|
|
echo "## Nightly Regression Summary" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "**Run ID:** ${{ needs.prepare.outputs.run_id }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "**Date:** ${{ needs.prepare.outputs.run_date }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "**Commit:** ${{ needs.prepare.outputs.commit_sha }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "### Job Results" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY
|
|
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Build | ${{ needs.build.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| PR-Gating Tests | ${{ needs.test-pr-gating.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Extended Tests | ${{ needs.test-extended.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Determinism | ${{ needs.determinism.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Cross-Module | ${{ needs.cross-module.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
echo "| Coverage | ${{ needs.coverage.result }} |" >> $GITHUB_STEP_SUMMARY
|
|
|
|
- name: Determine overall status
|
|
id: status
|
|
run: |
|
|
if [[ "${{ needs.build.result }}" == "failure" ]] || \
|
|
[[ "${{ needs.test-pr-gating.result }}" == "failure" ]] || \
|
|
[[ "${{ needs.determinism.result }}" == "failure" ]]; then
|
|
echo "status=failure" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "status=success" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
# Placeholder for notifications - configure webhook URL in secrets
|
|
- name: Send failure notification
|
|
if: steps.status.outputs.status == 'failure'
|
|
run: |
|
|
echo "::warning::Nightly regression failed - notification would be sent here"
|
|
# Uncomment and configure when webhook is available:
|
|
# curl -X POST "${{ secrets.SLACK_WEBHOOK_URL }}" \
|
|
# -H "Content-Type: application/json" \
|
|
# -d '{
|
|
# "text": "Nightly Regression Failed",
|
|
# "attachments": [{
|
|
# "color": "danger",
|
|
# "fields": [
|
|
# {"title": "Run ID", "value": "${{ needs.prepare.outputs.run_id }}", "short": true},
|
|
# {"title": "Commit", "value": "${{ needs.prepare.outputs.commit_sha }}", "short": true}
|
|
# ]
|
|
# }]
|
|
# }'
|
|
|
|
- name: Send success notification
|
|
if: steps.status.outputs.status == 'success' && github.event.inputs.notify_on_success == 'true'
|
|
run: |
|
|
echo "::notice::Nightly regression passed"
|
|
|
|
- name: Exit with appropriate code
|
|
if: steps.status.outputs.status == 'failure'
|
|
run: exit 1
|