up
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Signals CI & Image / signals-ci (push) Has been cancelled
Signals Reachability Scoring & Events / reachability-smoke (push) Has been cancelled
Signals Reachability Scoring & Events / sign-and-upload (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Reachability Corpus Validation / validate-corpus (push) Has been cancelled
Reachability Corpus Validation / validate-ground-truths (push) Has been cancelled
Scanner Analyzers / Discover Analyzers (push) Has been cancelled
Scanner Analyzers / Validate Test Fixtures (push) Has been cancelled
Reachability Corpus Validation / determinism-check (push) Has been cancelled
Scanner Analyzers / Build Analyzers (push) Has been cancelled
Scanner Analyzers / Test Language Analyzers (push) Has been cancelled
Scanner Analyzers / Verify Deterministic Output (push) Has been cancelled
Notify Smoke Test / Notify Unit Tests (push) Has been cancelled
Notify Smoke Test / Notifier Service Tests (push) Has been cancelled
Notify Smoke Test / Notification Smoke Test (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
This commit is contained in:
95
scripts/reachability/run_all.ps1
Normal file
95
scripts/reachability/run_all.ps1
Normal file
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# QA-CORPUS-401-031: Deterministic runner for reachability corpus tests (Windows)
|
||||
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(HelpMessage = "xUnit filter pattern (e.g., 'CorpusFixtureTests')")]
|
||||
[string]$Filter,
|
||||
|
||||
[Parameter(HelpMessage = "Test verbosity level")]
|
||||
[ValidateSet("quiet", "minimal", "normal", "detailed", "diagnostic")]
|
||||
[string]$Verbosity = "normal",
|
||||
|
||||
[Parameter(HelpMessage = "Build configuration")]
|
||||
[ValidateSet("Debug", "Release")]
|
||||
[string]$Configuration = "Release",
|
||||
|
||||
[Parameter(HelpMessage = "Skip build step")]
|
||||
[switch]$NoBuild
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$RepoRoot = (Resolve-Path (Join-Path $ScriptDir "..\..")).Path
|
||||
$TestProject = Join-Path $RepoRoot "tests\reachability\StellaOps.Reachability.FixtureTests\StellaOps.Reachability.FixtureTests.csproj"
|
||||
|
||||
function Write-LogInfo { param($Message) Write-Host "[INFO] $Message" -ForegroundColor Green }
|
||||
function Write-LogWarn { param($Message) Write-Host "[WARN] $Message" -ForegroundColor Yellow }
|
||||
function Write-LogError { param($Message) Write-Host "[ERROR] $Message" -ForegroundColor Red }
|
||||
|
||||
Write-LogInfo "Reachability Corpus Test Runner (Windows)"
|
||||
Write-LogInfo "Repository root: $RepoRoot"
|
||||
Write-LogInfo "Test project: $TestProject"
|
||||
|
||||
# Verify prerequisites
|
||||
$dotnetPath = Get-Command dotnet -ErrorAction SilentlyContinue
|
||||
if (-not $dotnetPath) {
|
||||
Write-LogError "dotnet CLI not found. Please install .NET SDK."
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Verify corpus exists
|
||||
$corpusManifest = Join-Path $RepoRoot "tests\reachability\corpus\manifest.json"
|
||||
if (-not (Test-Path $corpusManifest)) {
|
||||
Write-LogError "Corpus manifest not found at $corpusManifest"
|
||||
exit 1
|
||||
}
|
||||
|
||||
$reachbenchIndex = Join-Path $RepoRoot "tests\reachability\fixtures\reachbench-2025-expanded\INDEX.json"
|
||||
if (-not (Test-Path $reachbenchIndex)) {
|
||||
Write-LogError "Reachbench INDEX not found at $reachbenchIndex"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Build if needed
|
||||
if (-not $NoBuild) {
|
||||
Write-LogInfo "Building test project ($Configuration)..."
|
||||
& dotnet build $TestProject -c $Configuration --nologo
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-LogError "Build failed"
|
||||
exit $LASTEXITCODE
|
||||
}
|
||||
}
|
||||
|
||||
# Build test command arguments
|
||||
$testArgs = @(
|
||||
"test"
|
||||
$TestProject
|
||||
"-c"
|
||||
$Configuration
|
||||
"--no-build"
|
||||
"--verbosity"
|
||||
$Verbosity
|
||||
)
|
||||
|
||||
if ($Filter) {
|
||||
$testArgs += "--filter"
|
||||
$testArgs += "FullyQualifiedName~$Filter"
|
||||
Write-LogInfo "Running tests with filter: $Filter"
|
||||
} else {
|
||||
Write-LogInfo "Running all fixture tests..."
|
||||
}
|
||||
|
||||
# Run tests
|
||||
Write-LogInfo "Executing: dotnet $($testArgs -join ' ')"
|
||||
& dotnet @testArgs
|
||||
$exitCode = $LASTEXITCODE
|
||||
|
||||
if ($exitCode -eq 0) {
|
||||
Write-LogInfo "All tests passed!"
|
||||
} else {
|
||||
Write-LogError "Some tests failed (exit code: $exitCode)"
|
||||
}
|
||||
|
||||
exit $exitCode
|
||||
118
scripts/reachability/run_all.sh
Normal file
118
scripts/reachability/run_all.sh
Normal file
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# QA-CORPUS-401-031: Deterministic runner for reachability corpus tests
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
TEST_PROJECT="${REPO_ROOT}/tests/reachability/StellaOps.Reachability.FixtureTests/StellaOps.Reachability.FixtureTests.csproj"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $*"; }
|
||||
|
||||
# Parse arguments
|
||||
FILTER=""
|
||||
VERBOSITY="normal"
|
||||
CONFIGURATION="Release"
|
||||
NO_BUILD=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--filter)
|
||||
FILTER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--verbosity|-v)
|
||||
VERBOSITY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--configuration|-c)
|
||||
CONFIGURATION="$2"
|
||||
shift 2
|
||||
;;
|
||||
--no-build)
|
||||
NO_BUILD=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [options]"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " --filter <pattern> xUnit filter pattern (e.g., 'CorpusFixtureTests')"
|
||||
echo " --verbosity, -v <level> Test verbosity (quiet, minimal, normal, detailed, diagnostic)"
|
||||
echo " --configuration, -c Build configuration (Debug, Release)"
|
||||
echo " --no-build Skip build step"
|
||||
echo " --help, -h Show this help"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 # Run all fixture tests"
|
||||
echo " $0 --filter CorpusFixtureTests # Run only corpus tests"
|
||||
echo " $0 --filter ReachbenchFixtureTests # Run only reachbench tests"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
log_info "Reachability Corpus Test Runner"
|
||||
log_info "Repository root: ${REPO_ROOT}"
|
||||
log_info "Test project: ${TEST_PROJECT}"
|
||||
|
||||
# Verify prerequisites
|
||||
if ! command -v dotnet &> /dev/null; then
|
||||
log_error "dotnet CLI not found. Please install .NET SDK."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify corpus exists
|
||||
if [[ ! -f "${REPO_ROOT}/tests/reachability/corpus/manifest.json" ]]; then
|
||||
log_error "Corpus manifest not found at tests/reachability/corpus/manifest.json"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${REPO_ROOT}/tests/reachability/fixtures/reachbench-2025-expanded/INDEX.json" ]]; then
|
||||
log_error "Reachbench INDEX not found at tests/reachability/fixtures/reachbench-2025-expanded/INDEX.json"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build if needed
|
||||
if [[ "${NO_BUILD}" == false ]]; then
|
||||
log_info "Building test project (${CONFIGURATION})..."
|
||||
dotnet build "${TEST_PROJECT}" -c "${CONFIGURATION}" --nologo
|
||||
fi
|
||||
|
||||
# Build test command
|
||||
TEST_CMD="dotnet test ${TEST_PROJECT} -c ${CONFIGURATION} --no-build --verbosity ${VERBOSITY}"
|
||||
|
||||
if [[ -n "${FILTER}" ]]; then
|
||||
TEST_CMD="${TEST_CMD} --filter \"FullyQualifiedName~${FILTER}\""
|
||||
log_info "Running tests with filter: ${FILTER}"
|
||||
else
|
||||
log_info "Running all fixture tests..."
|
||||
fi
|
||||
|
||||
# Run tests
|
||||
log_info "Executing: ${TEST_CMD}"
|
||||
eval "${TEST_CMD}"
|
||||
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [[ ${EXIT_CODE} -eq 0 ]]; then
|
||||
log_info "All tests passed!"
|
||||
else
|
||||
log_error "Some tests failed (exit code: ${EXIT_CODE})"
|
||||
fi
|
||||
|
||||
exit ${EXIT_CODE}
|
||||
73
scripts/reachability/verify_corpus_hashes.sh
Normal file
73
scripts/reachability/verify_corpus_hashes.sh
Normal file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# QA-CORPUS-401-031: Verify SHA-256 hashes in corpus manifest
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
CORPUS_DIR="${REPO_ROOT}/tests/reachability/corpus"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $*"; }
|
||||
|
||||
cd "${CORPUS_DIR}"
|
||||
|
||||
if [[ ! -f "manifest.json" ]]; then
|
||||
log_error "manifest.json not found in ${CORPUS_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Verifying corpus hashes..."
|
||||
|
||||
# Use Python for JSON parsing (more portable than jq)
|
||||
python3 << 'PYTHON_SCRIPT'
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
|
||||
with open('manifest.json') as f:
|
||||
manifest = json.load(f)
|
||||
|
||||
errors = []
|
||||
verified = 0
|
||||
|
||||
for entry in manifest:
|
||||
case_id = entry['id']
|
||||
lang = entry['language']
|
||||
case_dir = os.path.join(lang, case_id)
|
||||
|
||||
if not os.path.isdir(case_dir):
|
||||
errors.append(f"{case_id}: case directory missing ({case_dir})")
|
||||
continue
|
||||
|
||||
for filename, expected_hash in entry['files'].items():
|
||||
filepath = os.path.join(case_dir, filename)
|
||||
|
||||
if not os.path.exists(filepath):
|
||||
errors.append(f"{case_id}: {filename} not found")
|
||||
continue
|
||||
|
||||
with open(filepath, 'rb') as f:
|
||||
actual_hash = hashlib.sha256(f.read()).hexdigest()
|
||||
|
||||
if actual_hash != expected_hash:
|
||||
errors.append(f"{case_id}: {filename} hash mismatch")
|
||||
errors.append(f" expected: {expected_hash}")
|
||||
errors.append(f" actual: {actual_hash}")
|
||||
else:
|
||||
verified += 1
|
||||
|
||||
if errors:
|
||||
print(f"\033[0;31m[ERROR]\033[0m Hash verification failed:")
|
||||
for err in errors:
|
||||
print(f" {err}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"\033[0;32m[INFO]\033[0m Verified {verified} files across {len(manifest)} corpus entries")
|
||||
sys.exit(0)
|
||||
PYTHON_SCRIPT
|
||||
Reference in New Issue
Block a user