Add determinism tests for verdict artifact generation and update SHA256 sums script
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
248
.gitea/workflows/connector-fixture-drift.yml
Normal file
248
.gitea/workflows/connector-fixture-drift.yml
Normal file
@@ -0,0 +1,248 @@
|
||||
# -----------------------------------------------------------------------------
|
||||
# connector-fixture-drift.yml
|
||||
# Sprint: SPRINT_5100_0007_0005_connector_fixtures
|
||||
# Task: CONN-FIX-016
|
||||
# Description: Weekly schema drift detection for connector fixtures with auto-PR
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
name: Connector Fixture Drift
|
||||
|
||||
on:
|
||||
# Weekly schedule: Sunday at 2:00 UTC
|
||||
schedule:
|
||||
- cron: '0 2 * * 0'
|
||||
# Manual trigger for on-demand drift detection
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
auto_update:
|
||||
description: 'Auto-update fixtures if drift detected'
|
||||
required: false
|
||||
default: 'true'
|
||||
type: boolean
|
||||
create_pr:
|
||||
description: 'Create PR for updated fixtures'
|
||||
required: false
|
||||
default: 'true'
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
DOTNET_NOLOGO: 1
|
||||
DOTNET_CLI_TELEMETRY_OPTOUT: 1
|
||||
TZ: UTC
|
||||
|
||||
jobs:
|
||||
detect-drift:
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
outputs:
|
||||
has_drift: ${{ steps.drift.outputs.has_drift }}
|
||||
drift_count: ${{ steps.drift.outputs.drift_count }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: '10.0.100'
|
||||
include-prerelease: true
|
||||
|
||||
- name: Cache NuGet packages
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.nuget/packages
|
||||
local-nugets/packages
|
||||
key: fixture-drift-nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj') }}
|
||||
|
||||
- name: Restore solution
|
||||
run: dotnet restore src/StellaOps.sln --configfile nuget.config
|
||||
|
||||
- name: Build test projects
|
||||
run: |
|
||||
dotnet build src/Concelier/__Tests/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj -c Release --no-restore
|
||||
dotnet build src/Excititor/__Tests/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.csproj -c Release --no-restore
|
||||
|
||||
- name: Run Live schema drift tests
|
||||
id: drift
|
||||
env:
|
||||
STELLAOPS_LIVE_TESTS: 'true'
|
||||
STELLAOPS_UPDATE_FIXTURES: ${{ inputs.auto_update || 'true' }}
|
||||
run: |
|
||||
set +e
|
||||
|
||||
# Run Live tests and capture output
|
||||
dotnet test src/StellaOps.sln \
|
||||
--filter "Category=Live" \
|
||||
--no-build \
|
||||
-c Release \
|
||||
--logger "console;verbosity=detailed" \
|
||||
--results-directory out/drift-results \
|
||||
2>&1 | tee out/drift-output.log
|
||||
|
||||
EXIT_CODE=$?
|
||||
|
||||
# Check for fixture changes
|
||||
CHANGED_FILES=$(git diff --name-only -- '**/Fixtures/*.json' '**/Expected/*.json' | wc -l)
|
||||
|
||||
if [ "$CHANGED_FILES" -gt 0 ]; then
|
||||
echo "has_drift=true" >> $GITHUB_OUTPUT
|
||||
echo "drift_count=$CHANGED_FILES" >> $GITHUB_OUTPUT
|
||||
echo "::warning::Schema drift detected in $CHANGED_FILES fixture files"
|
||||
else
|
||||
echo "has_drift=false" >> $GITHUB_OUTPUT
|
||||
echo "drift_count=0" >> $GITHUB_OUTPUT
|
||||
echo "::notice::No schema drift detected"
|
||||
fi
|
||||
|
||||
# Don't fail workflow on test failures (drift is expected)
|
||||
exit 0
|
||||
|
||||
- name: Show changed fixtures
|
||||
if: steps.drift.outputs.has_drift == 'true'
|
||||
run: |
|
||||
echo "## Changed fixture files:"
|
||||
git diff --name-only -- '**/Fixtures/*.json' '**/Expected/*.json'
|
||||
echo ""
|
||||
echo "## Diff summary:"
|
||||
git diff --stat -- '**/Fixtures/*.json' '**/Expected/*.json'
|
||||
|
||||
- name: Upload drift report
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: drift-report-${{ github.run_id }}
|
||||
path: |
|
||||
out/drift-output.log
|
||||
out/drift-results/**
|
||||
retention-days: 30
|
||||
|
||||
create-pr:
|
||||
needs: detect-drift
|
||||
if: needs.detect-drift.outputs.has_drift == 'true' && (github.event.inputs.create_pr == 'true' || github.event_name == 'schedule')
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
dotnet-version: '10.0.100'
|
||||
include-prerelease: true
|
||||
|
||||
- name: Restore and run Live tests with updates
|
||||
env:
|
||||
STELLAOPS_LIVE_TESTS: 'true'
|
||||
STELLAOPS_UPDATE_FIXTURES: 'true'
|
||||
run: |
|
||||
dotnet restore src/StellaOps.sln --configfile nuget.config
|
||||
dotnet test src/StellaOps.sln \
|
||||
--filter "Category=Live" \
|
||||
-c Release \
|
||||
--logger "console;verbosity=minimal" \
|
||||
|| true
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config user.name "StellaOps Bot"
|
||||
git config user.email "bot@stellaops.local"
|
||||
|
||||
- name: Create branch and commit
|
||||
id: commit
|
||||
run: |
|
||||
BRANCH_NAME="fixture-drift/$(date +%Y-%m-%d)"
|
||||
echo "branch=$BRANCH_NAME" >> $GITHUB_OUTPUT
|
||||
|
||||
# Check for changes
|
||||
if git diff --quiet -- '**/Fixtures/*.json' '**/Expected/*.json'; then
|
||||
echo "No fixture changes to commit"
|
||||
echo "has_changes=false" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||||
|
||||
# Create branch
|
||||
git checkout -b "$BRANCH_NAME"
|
||||
|
||||
# Stage fixture changes
|
||||
git add '**/Fixtures/*.json' '**/Expected/*.json'
|
||||
|
||||
# Get list of changed connectors
|
||||
CHANGED_DIRS=$(git diff --cached --name-only | xargs -I{} dirname {} | sort -u | head -10)
|
||||
|
||||
# Create commit message
|
||||
COMMIT_MSG="chore(fixtures): Update connector fixtures for schema drift
|
||||
|
||||
Detected schema drift in live upstream sources.
|
||||
Updated fixture files to match current API responses.
|
||||
|
||||
Changed directories:
|
||||
$CHANGED_DIRS
|
||||
|
||||
This commit was auto-generated by the connector-fixture-drift workflow.
|
||||
|
||||
🤖 Generated with [StellaOps CI](https://stellaops.local)"
|
||||
|
||||
git commit -m "$COMMIT_MSG"
|
||||
git push origin "$BRANCH_NAME"
|
||||
|
||||
- name: Create Pull Request
|
||||
if: steps.commit.outputs.has_changes == 'true'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const branch = '${{ steps.commit.outputs.branch }}';
|
||||
const driftCount = '${{ needs.detect-drift.outputs.drift_count }}';
|
||||
|
||||
const { data: pr } = await github.rest.pulls.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: `chore(fixtures): Update ${driftCount} connector fixtures for schema drift`,
|
||||
head: branch,
|
||||
base: 'main',
|
||||
body: `## Summary
|
||||
|
||||
Automated fixture update due to schema drift detected in live upstream sources.
|
||||
|
||||
- **Fixtures Updated**: ${driftCount}
|
||||
- **Detection Date**: ${new Date().toISOString().split('T')[0]}
|
||||
- **Workflow Run**: [#${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
||||
|
||||
## Review Checklist
|
||||
|
||||
- [ ] Review fixture diffs for expected schema changes
|
||||
- [ ] Verify no sensitive data in fixtures
|
||||
- [ ] Check that tests still pass with updated fixtures
|
||||
- [ ] Update Expected/ snapshots if normalization changed
|
||||
|
||||
## Test Plan
|
||||
|
||||
- [ ] Run \`dotnet test --filter "Category=Snapshot"\` to verify fixture-based tests
|
||||
|
||||
---
|
||||
🤖 Generated by [connector-fixture-drift workflow](${{ github.server_url }}/${{ github.repository }}/actions/workflows/connector-fixture-drift.yml)
|
||||
`
|
||||
});
|
||||
|
||||
console.log(`Created PR #${pr.number}: ${pr.html_url}`);
|
||||
|
||||
// Add labels
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: pr.number,
|
||||
labels: ['automated', 'fixtures', 'schema-drift']
|
||||
});
|
||||
Reference in New Issue
Block a user