#!/usr/bin/env bash # ============================================================================= # compute-ttfs-metrics.sh # Computes Time-to-First-Signal (TTFS) metrics from test runs # # Usage: ./compute-ttfs-metrics.sh [options] # --results-path PATH Path to test results directory # --output FILE Output JSON file (default: stdout) # --baseline FILE Baseline TTFS file for comparison # --dry-run Show what would be computed # --strict Exit non-zero if thresholds are violated # --verbose Enable verbose output # # Output: JSON with TTFS p50, p95, p99 metrics and regression status # ============================================================================= set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" # Default paths RESULTS_PATH="${REPO_ROOT}/bench/results" OUTPUT_FILE="" BASELINE_FILE="${REPO_ROOT}/bench/baselines/ttfs-baseline.json" DRY_RUN=false STRICT=false VERBOSE=false # Parse arguments while [[ $# -gt 0 ]]; do case "$1" in --results-path) RESULTS_PATH="$2" shift 2 ;; --output) OUTPUT_FILE="$2" shift 2 ;; --baseline) BASELINE_FILE="$2" shift 2 ;; --dry-run) DRY_RUN=true shift ;; --strict) STRICT=true shift ;; --verbose) VERBOSE=true shift ;; -h|--help) head -20 "$0" | tail -15 exit 0 ;; *) echo "Unknown option: $1" >&2 exit 1 ;; esac done log() { if [[ "${VERBOSE}" == "true" ]]; then echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" >&2 fi } error() { echo "[ERROR] $*" >&2 } warn() { echo "[WARN] $*" >&2 } # Calculate percentiles from sorted array percentile() { local -n arr=$1 local p=$2 local n=${#arr[@]} if [[ $n -eq 0 ]]; then echo "0" return fi local idx=$(echo "scale=0; ($n - 1) * $p / 100" | bc) echo "${arr[$idx]}" } if [[ "${DRY_RUN}" == "true" ]]; then log "[DRY RUN] Would process TTFS metrics..." cat </dev/null || true) SCAN_TYPE=$(jq -r '.scan_type // "unknown"' "${result_file}" 2>/dev/null || echo "unknown") if [[ -n "${TTFS}" ]] && [[ "${TTFS}" != "null" ]]; then ttfs_values+=("${TTFS}") case "${SCAN_TYPE}" in image|image_scan|container) image_ttfs+=("${TTFS}") ;; filesystem|fs|fs_scan) fs_ttfs+=("${TTFS}") ;; sbom|sbom_scan) sbom_ttfs+=("${TTFS}") ;; esac fi done # Sort arrays for percentile calculation IFS=$'\n' ttfs_sorted=($(sort -n <<<"${ttfs_values[*]}")); unset IFS IFS=$'\n' image_sorted=($(sort -n <<<"${image_ttfs[*]}")); unset IFS IFS=$'\n' fs_sorted=($(sort -n <<<"${fs_ttfs[*]}")); unset IFS IFS=$'\n' sbom_sorted=($(sort -n <<<"${sbom_ttfs[*]}")); unset IFS # Calculate overall metrics SAMPLE_COUNT=${#ttfs_values[@]} if [[ $SAMPLE_COUNT -eq 0 ]]; then warn "No TTFS samples found" P50=0 P95=0 P99=0 MIN=0 MAX=0 MEAN=0 else P50=$(percentile ttfs_sorted 50) P95=$(percentile ttfs_sorted 95) P99=$(percentile ttfs_sorted 99) MIN=${ttfs_sorted[0]} MAX=${ttfs_sorted[-1]} # Calculate mean SUM=0 for v in "${ttfs_values[@]}"; do SUM=$((SUM + v)) done MEAN=$((SUM / SAMPLE_COUNT)) fi # Calculate per-type metrics IMAGE_P50=$(percentile image_sorted 50) IMAGE_P95=$(percentile image_sorted 95) IMAGE_P99=$(percentile image_sorted 99) FS_P50=$(percentile fs_sorted 50) FS_P95=$(percentile fs_sorted 95) FS_P99=$(percentile fs_sorted 99) SBOM_P50=$(percentile sbom_sorted 50) SBOM_P95=$(percentile sbom_sorted 95) SBOM_P99=$(percentile sbom_sorted 99) # Compare against baseline if available REGRESSION_DETECTED=false P50_REGRESSION_PCT=0 P95_REGRESSION_PCT=0 if [[ -f "${BASELINE_FILE}" ]]; then log "Comparing against baseline: ${BASELINE_FILE}" BASELINE_P50=$(jq -r '.metrics.ttfs_ms.p50 // 0' "${BASELINE_FILE}") BASELINE_P95=$(jq -r '.metrics.ttfs_ms.p95 // 0' "${BASELINE_FILE}") if [[ $BASELINE_P50 -gt 0 ]]; then P50_REGRESSION_PCT=$(echo "scale=2; (${P50} - ${BASELINE_P50}) * 100 / ${BASELINE_P50}" | bc) fi if [[ $BASELINE_P95 -gt 0 ]]; then P95_REGRESSION_PCT=$(echo "scale=2; (${P95} - ${BASELINE_P95}) * 100 / ${BASELINE_P95}" | bc) fi # Check for regression (>10% increase) if (( $(echo "${P50_REGRESSION_PCT} > 10" | bc -l) )) || (( $(echo "${P95_REGRESSION_PCT} > 10" | bc -l) )); then REGRESSION_DETECTED=true warn "TTFS regression detected: p50=${P50_REGRESSION_PCT}%, p95=${P95_REGRESSION_PCT}%" fi fi # Generate output OUTPUT=$(cat < "${OUTPUT_FILE}" log "Results written to ${OUTPUT_FILE}" else echo "${OUTPUT}" fi # Strict mode: fail on regression if [[ "${STRICT}" == "true" ]] && [[ "${REGRESSION_DETECTED}" == "true" ]]; then error "TTFS regression exceeds threshold" exit 1 fi exit 0