474 lines
18 KiB
YAML
474 lines
18 KiB
YAML
# =============================================================================
|
|
# e2e-reproducibility.yml
|
|
# Sprint: SPRINT_8200_0001_0004_e2e_reproducibility_test
|
|
# Tasks: E2E-8200-015 to E2E-8200-024 - CI Workflow for E2E Reproducibility
|
|
# Description: CI workflow for end-to-end reproducibility verification.
|
|
# Runs tests across multiple platforms and compares results.
|
|
# =============================================================================
|
|
|
|
name: E2E Reproducibility
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- 'src/**'
|
|
- 'tests/integration/StellaOps.Integration.E2E/**'
|
|
- 'tests/fixtures/**'
|
|
- '.gitea/workflows/e2e-reproducibility.yml'
|
|
push:
|
|
branches:
|
|
- main
|
|
- develop
|
|
paths:
|
|
- 'src/**'
|
|
- 'tests/integration/StellaOps.Integration.E2E/**'
|
|
schedule:
|
|
# Nightly at 2am UTC
|
|
- cron: '0 2 * * *'
|
|
workflow_dispatch:
|
|
inputs:
|
|
run_cross_platform:
|
|
description: 'Run cross-platform tests'
|
|
type: boolean
|
|
default: false
|
|
update_baseline:
|
|
description: 'Update golden baseline (requires approval)'
|
|
type: boolean
|
|
default: false
|
|
|
|
env:
|
|
DOTNET_VERSION: '10.0.x'
|
|
DOTNET_NOLOGO: true
|
|
DOTNET_CLI_TELEMETRY_OPTOUT: true
|
|
|
|
jobs:
|
|
# =============================================================================
|
|
# Job: Run E2E reproducibility tests on primary platform
|
|
# =============================================================================
|
|
reproducibility-ubuntu:
|
|
name: E2E Reproducibility (Ubuntu)
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
verdict_hash: ${{ steps.run-tests.outputs.verdict_hash }}
|
|
manifest_hash: ${{ steps.run-tests.outputs.manifest_hash }}
|
|
envelope_hash: ${{ steps.run-tests.outputs.envelope_hash }}
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:16-alpine
|
|
env:
|
|
POSTGRES_USER: test_user
|
|
POSTGRES_PASSWORD: test_password
|
|
POSTGRES_DB: stellaops_e2e_test
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
|
|
- name: Restore dependencies
|
|
run: dotnet restore tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj
|
|
|
|
- name: Build E2E tests
|
|
run: dotnet build tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj --no-restore -c Release
|
|
|
|
- name: Run E2E reproducibility tests
|
|
id: run-tests
|
|
run: |
|
|
dotnet test tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj \
|
|
--no-build \
|
|
-c Release \
|
|
--logger "trx;LogFileName=e2e-results.trx" \
|
|
--logger "console;verbosity=detailed" \
|
|
--results-directory ./TestResults \
|
|
-- RunConfiguration.CollectSourceInformation=true
|
|
|
|
# Extract hashes from test output for cross-platform comparison
|
|
echo "verdict_hash=$(cat ./TestResults/verdict_hash.txt 2>/dev/null || echo 'NOT_FOUND')" >> $GITHUB_OUTPUT
|
|
echo "manifest_hash=$(cat ./TestResults/manifest_hash.txt 2>/dev/null || echo 'NOT_FOUND')" >> $GITHUB_OUTPUT
|
|
echo "envelope_hash=$(cat ./TestResults/envelope_hash.txt 2>/dev/null || echo 'NOT_FOUND')" >> $GITHUB_OUTPUT
|
|
env:
|
|
ConnectionStrings__ScannerDb: "Host=localhost;Port=5432;Database=stellaops_e2e_test;Username=test_user;Password=test_password"
|
|
|
|
- name: Upload test results
|
|
uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: e2e-results-ubuntu
|
|
path: ./TestResults/
|
|
retention-days: 14
|
|
|
|
- name: Upload hash artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: hashes-ubuntu
|
|
path: |
|
|
./TestResults/verdict_hash.txt
|
|
./TestResults/manifest_hash.txt
|
|
./TestResults/envelope_hash.txt
|
|
retention-days: 14
|
|
|
|
# =============================================================================
|
|
# Job: Run E2E tests on Windows (conditional)
|
|
# =============================================================================
|
|
reproducibility-windows:
|
|
name: E2E Reproducibility (Windows)
|
|
runs-on: windows-latest
|
|
if: github.event_name == 'schedule' || github.event.inputs.run_cross_platform == 'true'
|
|
outputs:
|
|
verdict_hash: ${{ steps.run-tests.outputs.verdict_hash }}
|
|
manifest_hash: ${{ steps.run-tests.outputs.manifest_hash }}
|
|
envelope_hash: ${{ steps.run-tests.outputs.envelope_hash }}
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
|
|
- name: Restore dependencies
|
|
run: dotnet restore tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj
|
|
|
|
- name: Build E2E tests
|
|
run: dotnet build tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj --no-restore -c Release
|
|
|
|
- name: Run E2E reproducibility tests
|
|
id: run-tests
|
|
run: |
|
|
dotnet test tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj `
|
|
--no-build `
|
|
-c Release `
|
|
--logger "trx;LogFileName=e2e-results.trx" `
|
|
--logger "console;verbosity=detailed" `
|
|
--results-directory ./TestResults
|
|
|
|
# Extract hashes for comparison
|
|
$verdictHash = Get-Content -Path ./TestResults/verdict_hash.txt -ErrorAction SilentlyContinue
|
|
$manifestHash = Get-Content -Path ./TestResults/manifest_hash.txt -ErrorAction SilentlyContinue
|
|
$envelopeHash = Get-Content -Path ./TestResults/envelope_hash.txt -ErrorAction SilentlyContinue
|
|
|
|
"verdict_hash=$($verdictHash ?? 'NOT_FOUND')" >> $env:GITHUB_OUTPUT
|
|
"manifest_hash=$($manifestHash ?? 'NOT_FOUND')" >> $env:GITHUB_OUTPUT
|
|
"envelope_hash=$($envelopeHash ?? 'NOT_FOUND')" >> $env:GITHUB_OUTPUT
|
|
|
|
- name: Upload test results
|
|
uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: e2e-results-windows
|
|
path: ./TestResults/
|
|
retention-days: 14
|
|
|
|
- name: Upload hash artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: hashes-windows
|
|
path: |
|
|
./TestResults/verdict_hash.txt
|
|
./TestResults/manifest_hash.txt
|
|
./TestResults/envelope_hash.txt
|
|
retention-days: 14
|
|
|
|
# =============================================================================
|
|
# Job: Run E2E tests on macOS (conditional)
|
|
# =============================================================================
|
|
reproducibility-macos:
|
|
name: E2E Reproducibility (macOS)
|
|
runs-on: macos-latest
|
|
if: github.event_name == 'schedule' || github.event.inputs.run_cross_platform == 'true'
|
|
outputs:
|
|
verdict_hash: ${{ steps.run-tests.outputs.verdict_hash }}
|
|
manifest_hash: ${{ steps.run-tests.outputs.manifest_hash }}
|
|
envelope_hash: ${{ steps.run-tests.outputs.envelope_hash }}
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
|
|
- name: Restore dependencies
|
|
run: dotnet restore tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj
|
|
|
|
- name: Build E2E tests
|
|
run: dotnet build tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj --no-restore -c Release
|
|
|
|
- name: Run E2E reproducibility tests
|
|
id: run-tests
|
|
run: |
|
|
dotnet test tests/integration/StellaOps.Integration.E2E/StellaOps.Integration.E2E.csproj \
|
|
--no-build \
|
|
-c Release \
|
|
--logger "trx;LogFileName=e2e-results.trx" \
|
|
--logger "console;verbosity=detailed" \
|
|
--results-directory ./TestResults
|
|
|
|
# Extract hashes for comparison
|
|
echo "verdict_hash=$(cat ./TestResults/verdict_hash.txt 2>/dev/null || echo 'NOT_FOUND')" >> $GITHUB_OUTPUT
|
|
echo "manifest_hash=$(cat ./TestResults/manifest_hash.txt 2>/dev/null || echo 'NOT_FOUND')" >> $GITHUB_OUTPUT
|
|
echo "envelope_hash=$(cat ./TestResults/envelope_hash.txt 2>/dev/null || echo 'NOT_FOUND')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Upload test results
|
|
uses: actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: e2e-results-macos
|
|
path: ./TestResults/
|
|
retention-days: 14
|
|
|
|
- name: Upload hash artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: hashes-macos
|
|
path: |
|
|
./TestResults/verdict_hash.txt
|
|
./TestResults/manifest_hash.txt
|
|
./TestResults/envelope_hash.txt
|
|
retention-days: 14
|
|
|
|
# =============================================================================
|
|
# Job: Cross-platform hash comparison
|
|
# =============================================================================
|
|
cross-platform-compare:
|
|
name: Cross-Platform Hash Comparison
|
|
runs-on: ubuntu-latest
|
|
needs: [reproducibility-ubuntu, reproducibility-windows, reproducibility-macos]
|
|
if: always() && (github.event_name == 'schedule' || github.event.inputs.run_cross_platform == 'true')
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Download Ubuntu hashes
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: hashes-ubuntu
|
|
path: ./hashes/ubuntu
|
|
|
|
- name: Download Windows hashes
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: hashes-windows
|
|
path: ./hashes/windows
|
|
continue-on-error: true
|
|
|
|
- name: Download macOS hashes
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: hashes-macos
|
|
path: ./hashes/macos
|
|
continue-on-error: true
|
|
|
|
- name: Compare hashes across platforms
|
|
run: |
|
|
echo "=== Cross-Platform Hash Comparison ==="
|
|
echo ""
|
|
|
|
ubuntu_verdict=$(cat ./hashes/ubuntu/verdict_hash.txt 2>/dev/null || echo "NOT_AVAILABLE")
|
|
windows_verdict=$(cat ./hashes/windows/verdict_hash.txt 2>/dev/null || echo "NOT_AVAILABLE")
|
|
macos_verdict=$(cat ./hashes/macos/verdict_hash.txt 2>/dev/null || echo "NOT_AVAILABLE")
|
|
|
|
echo "Verdict Hashes:"
|
|
echo " Ubuntu: $ubuntu_verdict"
|
|
echo " Windows: $windows_verdict"
|
|
echo " macOS: $macos_verdict"
|
|
echo ""
|
|
|
|
ubuntu_manifest=$(cat ./hashes/ubuntu/manifest_hash.txt 2>/dev/null || echo "NOT_AVAILABLE")
|
|
windows_manifest=$(cat ./hashes/windows/manifest_hash.txt 2>/dev/null || echo "NOT_AVAILABLE")
|
|
macos_manifest=$(cat ./hashes/macos/manifest_hash.txt 2>/dev/null || echo "NOT_AVAILABLE")
|
|
|
|
echo "Manifest Hashes:"
|
|
echo " Ubuntu: $ubuntu_manifest"
|
|
echo " Windows: $windows_manifest"
|
|
echo " macOS: $macos_manifest"
|
|
echo ""
|
|
|
|
# Check if all available hashes match
|
|
all_match=true
|
|
|
|
if [ "$ubuntu_verdict" != "NOT_AVAILABLE" ] && [ "$windows_verdict" != "NOT_AVAILABLE" ]; then
|
|
if [ "$ubuntu_verdict" != "$windows_verdict" ]; then
|
|
echo "❌ FAIL: Ubuntu and Windows verdict hashes differ!"
|
|
all_match=false
|
|
fi
|
|
fi
|
|
|
|
if [ "$ubuntu_verdict" != "NOT_AVAILABLE" ] && [ "$macos_verdict" != "NOT_AVAILABLE" ]; then
|
|
if [ "$ubuntu_verdict" != "$macos_verdict" ]; then
|
|
echo "❌ FAIL: Ubuntu and macOS verdict hashes differ!"
|
|
all_match=false
|
|
fi
|
|
fi
|
|
|
|
if [ "$all_match" = true ]; then
|
|
echo "✅ All available platform hashes match!"
|
|
else
|
|
echo ""
|
|
echo "Cross-platform reproducibility verification FAILED."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Create comparison report
|
|
run: |
|
|
cat > ./cross-platform-report.md << 'EOF'
|
|
# Cross-Platform Reproducibility Report
|
|
|
|
## Test Run Information
|
|
- **Workflow Run:** ${{ github.run_id }}
|
|
- **Trigger:** ${{ github.event_name }}
|
|
- **Commit:** ${{ github.sha }}
|
|
- **Branch:** ${{ github.ref_name }}
|
|
|
|
## Hash Comparison
|
|
|
|
| Platform | Verdict Hash | Manifest Hash | Status |
|
|
|----------|--------------|---------------|--------|
|
|
| Ubuntu | ${{ needs.reproducibility-ubuntu.outputs.verdict_hash }} | ${{ needs.reproducibility-ubuntu.outputs.manifest_hash }} | ✅ |
|
|
| Windows | ${{ needs.reproducibility-windows.outputs.verdict_hash }} | ${{ needs.reproducibility-windows.outputs.manifest_hash }} | ${{ needs.reproducibility-windows.result == 'success' && '✅' || '⚠️' }} |
|
|
| macOS | ${{ needs.reproducibility-macos.outputs.verdict_hash }} | ${{ needs.reproducibility-macos.outputs.manifest_hash }} | ${{ needs.reproducibility-macos.result == 'success' && '✅' || '⚠️' }} |
|
|
|
|
## Conclusion
|
|
|
|
Cross-platform reproducibility: **${{ job.status == 'success' && 'VERIFIED' || 'NEEDS REVIEW' }}**
|
|
EOF
|
|
|
|
cat ./cross-platform-report.md
|
|
|
|
- name: Upload comparison report
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: cross-platform-report
|
|
path: ./cross-platform-report.md
|
|
retention-days: 30
|
|
|
|
# =============================================================================
|
|
# Job: Golden baseline comparison
|
|
# =============================================================================
|
|
golden-baseline:
|
|
name: Golden Baseline Verification
|
|
runs-on: ubuntu-latest
|
|
needs: [reproducibility-ubuntu]
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Download current hashes
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: hashes-ubuntu
|
|
path: ./current
|
|
|
|
- name: Compare with golden baseline
|
|
run: |
|
|
echo "=== Golden Baseline Comparison ==="
|
|
|
|
baseline_file="./bench/determinism/golden-baseline/e2e-hashes.json"
|
|
|
|
if [ ! -f "$baseline_file" ]; then
|
|
echo "⚠️ Golden baseline not found. Skipping comparison."
|
|
echo "To create baseline, run with update_baseline=true"
|
|
exit 0
|
|
fi
|
|
|
|
current_verdict=$(cat ./current/verdict_hash.txt 2>/dev/null || echo "NOT_FOUND")
|
|
baseline_verdict=$(jq -r '.verdict_hash' "$baseline_file" 2>/dev/null || echo "NOT_FOUND")
|
|
|
|
echo "Current verdict hash: $current_verdict"
|
|
echo "Baseline verdict hash: $baseline_verdict"
|
|
|
|
if [ "$current_verdict" != "$baseline_verdict" ]; then
|
|
echo ""
|
|
echo "❌ FAIL: Current run does not match golden baseline!"
|
|
echo ""
|
|
echo "This may indicate:"
|
|
echo " 1. An intentional change requiring baseline update"
|
|
echo " 2. An unintentional regression in reproducibility"
|
|
echo ""
|
|
echo "To update baseline, run workflow with update_baseline=true"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "✅ Current run matches golden baseline!"
|
|
|
|
- name: Update golden baseline (if requested)
|
|
if: github.event.inputs.update_baseline == 'true'
|
|
run: |
|
|
mkdir -p ./bench/determinism/golden-baseline
|
|
|
|
cat > ./bench/determinism/golden-baseline/e2e-hashes.json << EOF
|
|
{
|
|
"verdict_hash": "$(cat ./current/verdict_hash.txt 2>/dev/null || echo 'NOT_SET')",
|
|
"manifest_hash": "$(cat ./current/manifest_hash.txt 2>/dev/null || echo 'NOT_SET')",
|
|
"envelope_hash": "$(cat ./current/envelope_hash.txt 2>/dev/null || echo 'NOT_SET')",
|
|
"updated_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"updated_by": "${{ github.actor }}",
|
|
"commit": "${{ github.sha }}"
|
|
}
|
|
EOF
|
|
|
|
echo "Golden baseline updated:"
|
|
cat ./bench/determinism/golden-baseline/e2e-hashes.json
|
|
|
|
- name: Commit baseline update
|
|
if: github.event.inputs.update_baseline == 'true'
|
|
uses: stefanzweifel/git-auto-commit-action@v5
|
|
with:
|
|
commit_message: "chore: Update E2E reproducibility golden baseline"
|
|
file_pattern: bench/determinism/golden-baseline/e2e-hashes.json
|
|
|
|
# =============================================================================
|
|
# Job: Status check gate
|
|
# =============================================================================
|
|
reproducibility-gate:
|
|
name: Reproducibility Gate
|
|
runs-on: ubuntu-latest
|
|
needs: [reproducibility-ubuntu, golden-baseline]
|
|
if: always()
|
|
|
|
steps:
|
|
- name: Check reproducibility status
|
|
run: |
|
|
ubuntu_status="${{ needs.reproducibility-ubuntu.result }}"
|
|
baseline_status="${{ needs.golden-baseline.result }}"
|
|
|
|
echo "Ubuntu E2E tests: $ubuntu_status"
|
|
echo "Golden baseline: $baseline_status"
|
|
|
|
if [ "$ubuntu_status" != "success" ]; then
|
|
echo "❌ E2E reproducibility tests failed!"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$baseline_status" == "failure" ]; then
|
|
echo "⚠️ Golden baseline comparison failed (may require review)"
|
|
# Don't fail the gate for baseline mismatch - it may be intentional
|
|
fi
|
|
|
|
echo "✅ Reproducibility gate passed!"
|