5100* tests strengthtenen work

.gitea/workflows/parity-tests.yml

@@ -0,0 +1,186 @@
+name: Parity Tests
+
+# Parity testing workflow: compares StellaOps against competitor scanners
+# (Syft, Grype, Trivy) on a standardized fixture set.
+#
+# Schedule: Nightly at 02:00 UTC; Weekly full run on Sunday 00:00 UTC
+# NOT a PR gate - too slow and has external dependencies
+
+on:
+  schedule:
+    # Nightly at 02:00 UTC (quick fixture set)
+    - cron: '0 2 * * *'
+    # Weekly on Sunday at 00:00 UTC (full fixture set)
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
+    inputs:
+      fixture_set:
+        description: 'Fixture set to use'
+        required: false
+        default: 'quick'
+        type: choice
+        options:
+          - quick
+          - full
+      enable_drift_detection:
+        description: 'Enable drift detection analysis'
+        required: false
+        default: 'true'
+        type: boolean
+
+env:
+  DOTNET_VERSION: '10.0.x'
+  SYFT_VERSION: '1.9.0'
+  GRYPE_VERSION: '0.79.3'
+  TRIVY_VERSION: '0.54.1'
+  PARITY_RESULTS_PATH: 'bench/results/parity'
+
+jobs:
+  parity-tests:
+    name: Competitor Parity Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: ${{ env.DOTNET_VERSION }}
+
+      - name: Install Syft
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin v${{ env.SYFT_VERSION }}
+          syft version
+
+      - name: Install Grype
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin v${{ env.GRYPE_VERSION }}
+          grype version
+
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v${{ env.TRIVY_VERSION }}
+          trivy --version
+
+      - name: Determine fixture set
+        id: fixtures
+        run: |
+          # Weekly runs use full fixture set
+          if [[ "${{ github.event.schedule }}" == "0 0 * * 0" ]]; then
+            echo "fixture_set=full" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "fixture_set=${{ inputs.fixture_set }}" >> $GITHUB_OUTPUT
+          else
+            echo "fixture_set=quick" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build parity tests
+        run: |
+          dotnet build tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj -c Release
+
+      - name: Run parity tests
+        id: parity
+        run: |
+          mkdir -p ${{ env.PARITY_RESULTS_PATH }}
+          RUN_ID=$(date -u +%Y%m%dT%H%M%SZ)
+          echo "run_id=${RUN_ID}" >> $GITHUB_OUTPUT
+          
+          dotnet test tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj \
+            -c Release \
+            --no-build \
+            --logger "trx;LogFileName=parity-results.trx" \
+            --results-directory ${{ env.PARITY_RESULTS_PATH }} \
+            -e PARITY_FIXTURE_SET=${{ steps.fixtures.outputs.fixture_set }} \
+            -e PARITY_RUN_ID=${RUN_ID} \
+            -e PARITY_OUTPUT_PATH=${{ env.PARITY_RESULTS_PATH }} \
+            || true  # Don't fail workflow on test failures
+
+      - name: Store parity results
+        run: |
+          # Copy JSON results to time-series storage
+          if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json" ]; then
+            echo "Parity results stored successfully"
+            cat ${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json | jq .
+          else
+            echo "Warning: No parity results file found"
+          fi
+
+      - name: Run drift detection
+        if: ${{ github.event_name != 'workflow_dispatch' || inputs.enable_drift_detection == 'true' }}
+        run: |
+          # Analyze drift from historical results
+          dotnet run --project tests/parity/StellaOps.Parity.Tests/StellaOps.Parity.Tests.csproj \
+            --no-build \
+            -- analyze-drift \
+            --results-path ${{ env.PARITY_RESULTS_PATH }} \
+            --threshold 0.05 \
+            --trend-days 3 \
+            || true
+
+      - name: Upload parity results
+        uses: actions/upload-artifact@v4
+        with:
+          name: parity-results-${{ steps.parity.outputs.run_id }}
+          path: ${{ env.PARITY_RESULTS_PATH }}
+          retention-days: 90
+
+      - name: Export Prometheus metrics
+        if: ${{ env.PROMETHEUS_PUSH_GATEWAY != '' }}
+        env:
+          PROMETHEUS_PUSH_GATEWAY: ${{ secrets.PROMETHEUS_PUSH_GATEWAY }}
+        run: |
+          # Push metrics to Prometheus Push Gateway if configured
+          if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-metrics.txt" ]; then
+            curl -X POST \
+              -H "Content-Type: text/plain" \
+              --data-binary @${{ env.PARITY_RESULTS_PATH }}/parity-metrics.txt \
+              "${PROMETHEUS_PUSH_GATEWAY}/metrics/job/parity_tests/instance/${{ steps.parity.outputs.run_id }}"
+          fi
+
+      - name: Generate comparison report
+        run: |
+          echo "## Parity Test Results - ${{ steps.parity.outputs.run_id }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "**Fixture Set:** ${{ steps.fixtures.outputs.fixture_set }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Competitor Versions:**" >> $GITHUB_STEP_SUMMARY
+          echo "- Syft: ${{ env.SYFT_VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Grype: ${{ env.GRYPE_VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Trivy: ${{ env.TRIVY_VERSION }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          
+          if [ -f "${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json" ]; then
+            echo "### Metrics Summary" >> $GITHUB_STEP_SUMMARY
+            jq -r '
+              "| Metric | StellaOps | Grype | Trivy |",
+              "|--------|-----------|-------|-------|",
+              "| SBOM Packages | \(.sbomMetrics.stellaOpsPackageCount) | \(.sbomMetrics.syftPackageCount) | - |",
+              "| Vulnerability Recall | \(.vulnMetrics.recall | . * 100 | round / 100)% | - | - |",
+              "| Vulnerability F1 | \(.vulnMetrics.f1Score | . * 100 | round / 100)% | - | - |",
+              "| Latency P95 (ms) | \(.latencyMetrics.stellaOpsP95Ms | round) | \(.latencyMetrics.grypeP95Ms | round) | \(.latencyMetrics.trivyP95Ms | round) |"
+            ' ${{ env.PARITY_RESULTS_PATH }}/parity-${{ steps.parity.outputs.run_id }}.json >> $GITHUB_STEP_SUMMARY || echo "Could not parse results" >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Alert on critical drift
+        if: failure()
+        uses: slackapi/slack-github-action@v1.25.0
+        with:
+          payload: |
+            {
+              "text": "⚠️ Parity test drift detected",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "*Parity Test Alert*\nDrift detected in competitor comparison metrics.\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Results>"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+        continue-on-error: true