#!/usr/bin/env bash # run-benchmark.sh # Smart-Diff Benchmark Runner # Sprint: SPRINT_3850_0001_0001 set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BENCH_ROOT="$SCRIPT_DIR" RESULTS_DIR="$BENCH_ROOT/results/$(date -u +%Y%m%d_%H%M%S)" # Parse arguments BASELINE_MODE="naive" COMPARE_MODE="smart" VERBOSE=false while [[ $# -gt 0 ]]; do case $1 in --baseline) BASELINE_MODE="$2" shift 2 ;; --compare) COMPARE_MODE="$2" shift 2 ;; --verbose|-v) VERBOSE=true shift ;; *) echo "Unknown option: $1" exit 1 ;; esac done echo "╔════════════════════════════════════════════════╗" echo "║ Smart-Diff Benchmark Suite ║" echo "╚════════════════════════════════════════════════╝" echo "" echo "Configuration:" echo " Baseline mode: $BASELINE_MODE" echo " Compare mode: $COMPARE_MODE" echo " Results dir: $RESULTS_DIR" echo "" mkdir -p "$RESULTS_DIR" # Function to run a test case run_test_case() { local test_id="$1" local description="$2" local base_sbom="$3" local target_sbom="$4" local expected_file="$5" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Test: $test_id - $description" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" local start_time=$(date +%s%3N) # Run smart-diff if command -v dotnet &> /dev/null; then dotnet run --project "$SCRIPT_DIR/../../src/Scanner/__Libraries/StellaOps.Scanner.SmartDiff" -- \ --base "$base_sbom" \ --target "$target_sbom" \ --output "$RESULTS_DIR/$test_id.diff.json" \ --format json 2>/dev/null || true fi local end_time=$(date +%s%3N) local elapsed=$((end_time - start_time)) echo " Time: ${elapsed}ms" # Verify determinism by running twice if [ -f "$RESULTS_DIR/$test_id.diff.json" ]; then local hash1=$(sha256sum "$RESULTS_DIR/$test_id.diff.json" | cut -d' ' -f1) if command -v dotnet &> /dev/null; then dotnet run --project "$SCRIPT_DIR/../../src/Scanner/__Libraries/StellaOps.Scanner.SmartDiff" -- \ --base "$base_sbom" \ --target "$target_sbom" \ --output "$RESULTS_DIR/$test_id.diff.run2.json" \ --format json 2>/dev/null || true fi if [ -f "$RESULTS_DIR/$test_id.diff.run2.json" ]; then local hash2=$(sha256sum "$RESULTS_DIR/$test_id.diff.run2.json" | cut -d' ' -f1) if [ "$hash1" = "$hash2" ]; then echo " ✓ Determinism verified" else echo " ✗ Determinism FAILED (different hashes)" fi fi else echo " ⊘ Skipped (dotnet not available or project missing)" fi echo "" } # Test Case 1: Layer-Aware Diff (using fixtures) if [ -f "$BENCH_ROOT/fixtures/base-alpine-3.18.sbom.cdx.json" ]; then run_test_case "TC-001-layer-added" \ "Layer addition detection" \ "$BENCH_ROOT/fixtures/base-alpine-3.18.sbom.cdx.json" \ "$BENCH_ROOT/fixtures/base-alpine-3.19.sbom.cdx.json" \ "$BENCH_ROOT/fixtures/expected/tc001-layer-added.diff.json" else echo "ℹ️ Skipping TC-001: Fixtures not found" echo " Run './tools/generate-fixtures.sh' to create test fixtures" fi # Generate summary echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Summary" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Results saved to: $RESULTS_DIR" # Create summary JSON cat > "$RESULTS_DIR/summary.json" <