name: Parity Tests # Parity testing workflow: compares StellaOps against competitor scanners # (Syft, Grype, Trivy) on a standardized fixture set. # # Schedule: Nightly at 02:00 UTC; Weekly full run on Sunday 00:00 UTC # NOT a PR gate - too slow and has external dependencies on: schedule: # Nightly at 02:00 UTC (quick fixture set) - cron: '0 2 * * *' # Weekly on Sunday at 00:00 UTC (full fixture set) - cron: '0 0 * * 0' workflow_dispatch: inputs: fixture_set: description: 'Fixture set to use' required: false default: 'quick' type: choice options: - quick - full enable_drift_detection: description: 'Enable drift detection analysis' required: false default: 'true' type: boolean env: DOTNET_VERSION: '10.0.x' SYFT_VERSION: '1.9.0' GRYPE_VERSION: '0.79.3' TRIVY_VERSION: '0.54.1' PARITY_RESULTS_PATH: 'bench/results/parity' jobs: parity-tests: name: Competitor Parity Tests runs-on: ubuntu-latest timeout-minutes: 120 steps: - name: Checkout repository uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ env.DOTNET_VERSION }} - name: Install Syft run: | curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin v${{ env.SYFT_VERSION }} syft version - name: Install Grype run: | curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin v${{ env.GRYPE_VERSION }} grype version - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v${{ env.TRIVY_VERSION }} trivy --version - name: Determine fixture set id: fixtures run: | # Weekly runs use full fixture set if [[ "${{ github.event.schedule }}" == "0 0 * * 0" ]]; then echo "fixture_set=full" >> $GITHUB_OUTPUT elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "fixture_set=${{ inputs.fixture_set }}" >> $GITHUB_OUTPUT else echo "fixture_set=quick" >> $GITHUB_OUTPUT fi - name: Build parity tests run: | dotnet build tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj -c Release - name: Run parity tests id: parity run: | mkdir -p ${{ env.PARITY_RESULTS_PATH }} RUN_ID=$(date -u +%Y%m%dT%H%M%SZ) echo "run_id=${RUN_ID}" >> $GITHUB_OUTPUT dotnet test tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj \ -c Release \ --no-build \ --logger "trx;LogFileName=parity-results.trx" \ --results-directory ${{ env.PARITY_RESULTS_PATH }} \ -e PARITY_FIXTURE_SET=${{ steps.fixtures.outputs.fixture_set }} \ -e PARITY_RUN_ID=${RUN_ID} \ -e PARITY_OUTPUT_PATH=${{ env.PARITY_RESULTS_PATH }} \ || true # Don't fail workflow on test failures - name: Store parity results run: | # Copy JSON results to time-series storage if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json" ]; then echo "Parity results stored successfully" cat ${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json | jq . else echo "Warning: No parity results file found" fi - name: Run drift detection if: ${{ github.event_name != 'workflow_dispatch' || inputs.enable_drift_detection == 'true' }} run: | # Analyze drift from historical results dotnet run --project tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj \ --no-build \ -- analyze-drift \ --results-path ${{ env.PARITY_RESULTS_PATH }} \ --threshold 0.05 \ --trend-days 3 \ || true - name: Upload parity results uses: actions/upload-artifact@v4 with: name: parity-results-${{ steps.parity.outputs.run_id }} path: ${{ env.PARITY_RESULTS_PATH }} retention-days: 90 - name: Export Prometheus metrics if: ${{ env.PROMETHEUS_PUSH_GATEWAY != '' }} env: PROMETHEUS_PUSH_GATEWAY: ${{ secrets.PROMETHEUS_PUSH_GATEWAY }} run: | # Push metrics to Prometheus Push Gateway if configured if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-metrics.txt" ]; then curl -X POST \ -H "Content-Type: text/plain" \ --data-binary @${{ env.PARITY_RESULTS_PATH }}/parity-metrics.txt \ "${PROMETHEUS_PUSH_GATEWAY}/metrics/job/parity_tests/instance/${{ steps.parity.outputs.run_id }}" fi - name: Generate comparison report run: | echo "## Parity Test Results - ${{ steps.parity.outputs.run_id }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Fixture Set:** ${{ steps.fixtures.outputs.fixture_set }}" >> $GITHUB_STEP_SUMMARY echo "**Competitor Versions:**" >> $GITHUB_STEP_SUMMARY echo "- Syft: ${{ env.SYFT_VERSION }}" >> $GITHUB_STEP_SUMMARY echo "- Grype: ${{ env.GRYPE_VERSION }}" >> $GITHUB_STEP_SUMMARY echo "- Trivy: ${{ env.TRIVY_VERSION }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json" ]; then echo "### Metrics Summary" >> $GITHUB_STEP_SUMMARY jq -r ' "| Metric | StellaOps | Grype | Trivy |", "|--------|-----------|-------|-------|", "| SBOM Packages | \(.sbomMetrics.stellaOpsPackageCount) | \(.sbomMetrics.syftPackageCount) | - |", "| Vulnerability Recall | \(.vulnMetrics.recall | . * 100 | round / 100)% | - | - |", "| Vulnerability F1 | \(.vulnMetrics.f1Score | . * 100 | round / 100)% | - | - |", "| Latency P95 (ms) | \(.latencyMetrics.stellaOpsP95Ms | round) | \(.latencyMetrics.grypeP95Ms | round) | \(.latencyMetrics.trivyP95Ms | round) |" ' ${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json >> $GITHUB_STEP_SUMMARY || echo "Could not parse results" >> $GITHUB_STEP_SUMMARY fi - name: Alert on critical drift if: failure() uses: slackapi/slack-github-action@v1.25.0 with: payload: | { "text": "⚠️ Parity test drift detected", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", "text": "*Parity Test Alert*\nDrift detected in competitor comparison metrics.\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Results>" } } ] } env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK continue-on-error: true