187 lines
7.3 KiB
YAML
187 lines
7.3 KiB
YAML
name: Parity Tests
|
|
|
|
# Parity testing workflow: compares StellaOps against competitor scanners
|
|
# (Syft, Grype, Trivy) on a standardized fixture set.
|
|
#
|
|
# Schedule: Nightly at 02:00 UTC; Weekly full run on Sunday 00:00 UTC
|
|
# NOT a PR gate - too slow and has external dependencies
|
|
|
|
on:
|
|
schedule:
|
|
# Nightly at 02:00 UTC (quick fixture set)
|
|
- cron: '0 2 * * *'
|
|
# Weekly on Sunday at 00:00 UTC (full fixture set)
|
|
- cron: '0 0 * * 0'
|
|
workflow_dispatch:
|
|
inputs:
|
|
fixture_set:
|
|
description: 'Fixture set to use'
|
|
required: false
|
|
default: 'quick'
|
|
type: choice
|
|
options:
|
|
- quick
|
|
- full
|
|
enable_drift_detection:
|
|
description: 'Enable drift detection analysis'
|
|
required: false
|
|
default: 'true'
|
|
type: boolean
|
|
|
|
env:
|
|
DOTNET_VERSION: '10.0.x'
|
|
SYFT_VERSION: '1.9.0'
|
|
GRYPE_VERSION: '0.79.3'
|
|
TRIVY_VERSION: '0.54.1'
|
|
PARITY_RESULTS_PATH: 'bench/results/parity'
|
|
|
|
jobs:
|
|
parity-tests:
|
|
name: Competitor Parity Tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 120
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Setup .NET
|
|
uses: actions/setup-dotnet@v4
|
|
with:
|
|
dotnet-version: ${{ env.DOTNET_VERSION }}
|
|
|
|
- name: Install Syft
|
|
run: |
|
|
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin v${{ env.SYFT_VERSION }}
|
|
syft version
|
|
|
|
- name: Install Grype
|
|
run: |
|
|
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin v${{ env.GRYPE_VERSION }}
|
|
grype version
|
|
|
|
- name: Install Trivy
|
|
run: |
|
|
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v${{ env.TRIVY_VERSION }}
|
|
trivy --version
|
|
|
|
- name: Determine fixture set
|
|
id: fixtures
|
|
run: |
|
|
# Weekly runs use full fixture set
|
|
if [[ "${{ github.event.schedule }}" == "0 0 * * 0" ]]; then
|
|
echo "fixture_set=full" >> $GITHUB_OUTPUT
|
|
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
|
echo "fixture_set=${{ inputs.fixture_set }}" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "fixture_set=quick" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- name: Build parity tests
|
|
run: |
|
|
dotnet build tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj -c Release
|
|
|
|
- name: Run parity tests
|
|
id: parity
|
|
run: |
|
|
mkdir -p ${{ env.PARITY_RESULTS_PATH }}
|
|
RUN_ID=$(date -u +%Y%m%dT%H%M%SZ)
|
|
echo "run_id=${RUN_ID}" >> $GITHUB_OUTPUT
|
|
|
|
dotnet test tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj \
|
|
-c Release \
|
|
--no-build \
|
|
--logger "trx;LogFileName=parity-results.trx" \
|
|
--results-directory ${{ env.PARITY_RESULTS_PATH }} \
|
|
-e PARITY_FIXTURE_SET=${{ steps.fixtures.outputs.fixture_set }} \
|
|
-e PARITY_RUN_ID=${RUN_ID} \
|
|
-e PARITY_OUTPUT_PATH=${{ env.PARITY_RESULTS_PATH }} \
|
|
|| true # Don't fail workflow on test failures
|
|
|
|
- name: Store parity results
|
|
run: |
|
|
# Copy JSON results to time-series storage
|
|
if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json" ]; then
|
|
echo "Parity results stored successfully"
|
|
cat ${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json | jq .
|
|
else
|
|
echo "Warning: No parity results file found"
|
|
fi
|
|
|
|
- name: Run drift detection
|
|
if: ${{ github.event_name != 'workflow_dispatch' || inputs.enable_drift_detection == 'true' }}
|
|
run: |
|
|
# Analyze drift from historical results
|
|
dotnet run --project tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj \
|
|
--no-build \
|
|
-- analyze-drift \
|
|
--results-path ${{ env.PARITY_RESULTS_PATH }} \
|
|
--threshold 0.05 \
|
|
--trend-days 3 \
|
|
|| true
|
|
|
|
- name: Upload parity results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: parity-results-${{ steps.parity.outputs.run_id }}
|
|
path: ${{ env.PARITY_RESULTS_PATH }}
|
|
retention-days: 90
|
|
|
|
- name: Export Prometheus metrics
|
|
if: ${{ env.PROMETHEUS_PUSH_GATEWAY != '' }}
|
|
env:
|
|
PROMETHEUS_PUSH_GATEWAY: ${{ secrets.PROMETHEUS_PUSH_GATEWAY }}
|
|
run: |
|
|
# Push metrics to Prometheus Push Gateway if configured
|
|
if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-metrics.txt" ]; then
|
|
curl -X POST \
|
|
-H "Content-Type: text/plain" \
|
|
--data-binary @${{ env.PARITY_RESULTS_PATH }}/parity-metrics.txt \
|
|
"${PROMETHEUS_PUSH_GATEWAY}/metrics/job/parity_tests/instance/${{ steps.parity.outputs.run_id }}"
|
|
fi
|
|
|
|
- name: Generate comparison report
|
|
run: |
|
|
echo "## Parity Test Results - ${{ steps.parity.outputs.run_id }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "**Fixture Set:** ${{ steps.fixtures.outputs.fixture_set }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "**Competitor Versions:**" >> $GITHUB_STEP_SUMMARY
|
|
echo "- Syft: ${{ env.SYFT_VERSION }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "- Grype: ${{ env.GRYPE_VERSION }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "- Trivy: ${{ env.TRIVY_VERSION }}" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json" ]; then
|
|
echo "### Metrics Summary" >> $GITHUB_STEP_SUMMARY
|
|
jq -r '
|
|
"| Metric | StellaOps | Grype | Trivy |",
|
|
"|--------|-----------|-------|-------|",
|
|
"| SBOM Packages | \(.sbomMetrics.stellaOpsPackageCount) | \(.sbomMetrics.syftPackageCount) | - |",
|
|
"| Vulnerability Recall | \(.vulnMetrics.recall | . * 100 | round / 100)% | - | - |",
|
|
"| Vulnerability F1 | \(.vulnMetrics.f1Score | . * 100 | round / 100)% | - | - |",
|
|
"| Latency P95 (ms) | \(.latencyMetrics.stellaOpsP95Ms | round) | \(.latencyMetrics.grypeP95Ms | round) | \(.latencyMetrics.trivyP95Ms | round) |"
|
|
' ${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json >> $GITHUB_STEP_SUMMARY || echo "Could not parse results" >> $GITHUB_STEP_SUMMARY
|
|
fi
|
|
|
|
- name: Alert on critical drift
|
|
if: failure()
|
|
uses: slackapi/slack-github-action@v1.25.0
|
|
with:
|
|
payload: |
|
|
{
|
|
"text": "⚠️ Parity test drift detected",
|
|
"blocks": [
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "*Parity Test Alert*\nDrift detected in competitor comparison metrics.\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Results>"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
|
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
|
|
continue-on-error: true
|