Files
git.stella-ops.org/devops/scripts/local-ci.sh

819 lines
23 KiB
Bash

#!/usr/bin/env bash
# =============================================================================
# LOCAL CI RUNNER
# =============================================================================
# Unified local CI/CD testing runner for StellaOps.
#
# Usage:
# ./devops/scripts/local-ci.sh [mode] [options]
#
# Modes:
# smoke - Quick smoke test (unit tests only, ~2 min)
# pr - Full PR-gating suite (all required checks, ~15 min)
# module - Module-specific tests (auto-detect or specified)
# workflow - Simulate specific workflow via act
# release - Release simulation (dry-run)
# full - All tests including extended categories (~45 min)
#
# Options:
# --category <cat> Run specific test category
# --workflow <name> Specific workflow to simulate
# --module <name> Specific module to test
# --docker Force Docker execution
# --native Force native execution
# --act Force act execution
# --parallel <n> Parallel test runners (default: CPU count)
# --verbose Verbose output
# --dry-run Show what would run without executing
# --rebuild Force rebuild of CI Docker image
# --no-services Skip starting CI services
# --keep-services Don't stop services after tests
# --help Show this help message
#
# Examples:
# ./local-ci.sh smoke # Quick validation
# ./local-ci.sh pr # Full PR check
# ./local-ci.sh module --module Scanner # Test Scanner module
# ./local-ci.sh workflow --workflow test-matrix
# ./local-ci.sh release --dry-run
#
# =============================================================================
set -euo pipefail
# =============================================================================
# SCRIPT INITIALIZATION
# =============================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
export REPO_ROOT
# Source libraries
source "$SCRIPT_DIR/lib/ci-common.sh"
source "$SCRIPT_DIR/lib/ci-docker.sh"
source "$SCRIPT_DIR/lib/ci-web.sh" 2>/dev/null || true # Web testing utilities
# =============================================================================
# CONSTANTS
# =============================================================================
# Modes
MODE_SMOKE="smoke"
MODE_PR="pr"
MODE_MODULE="module"
MODE_WORKFLOW="workflow"
MODE_RELEASE="release"
MODE_FULL="full"
# Test categories
PR_GATING_CATEGORIES=(Unit Architecture Contract Integration Security Golden)
EXTENDED_CATEGORIES=(Performance Benchmark AirGap Chaos Determinism Resilience Observability)
ALL_CATEGORIES=("${PR_GATING_CATEGORIES[@]}" "${EXTENDED_CATEGORIES[@]}")
# Default configuration
RESULTS_DIR="$REPO_ROOT/out/local-ci"
TRX_DIR="$RESULTS_DIR/trx"
LOGS_DIR="$RESULTS_DIR/logs"
# =============================================================================
# CONFIGURATION
# =============================================================================
MODE=""
EXECUTION_ENGINE="" # docker, native, act
SPECIFIC_CATEGORY=""
SPECIFIC_MODULE=""
SPECIFIC_WORKFLOW=""
PARALLEL_JOBS=""
VERBOSE=false
DRY_RUN=false
REBUILD_IMAGE=false
SKIP_SERVICES=false
KEEP_SERVICES=false
# =============================================================================
# USAGE
# =============================================================================
usage() {
cat <<EOF
Usage: $(basename "$0") [mode] [options]
Modes:
smoke Quick smoke test (unit tests only, ~2 min)
pr Full PR-gating suite (all required checks, ~15 min)
module Module-specific tests (auto-detect or specified)
workflow Simulate specific workflow via act
release Release simulation (dry-run)
full All tests including extended categories (~45 min)
Options:
--category <cat> Run specific test category (${ALL_CATEGORIES[*]})
--workflow <name> Specific workflow to simulate (for workflow mode)
--module <name> Specific module to test (for module mode)
--docker Force Docker execution
--native Force native execution
--act Force act execution
--parallel <n> Parallel test runners (default: auto-detect)
--verbose Verbose output
--dry-run Show what would run without executing
--rebuild Force rebuild of CI Docker image
--no-services Skip starting CI services
--keep-services Don't stop services after tests
--help Show this help message
Examples:
$(basename "$0") smoke # Quick validation before push
$(basename "$0") pr # Full PR check
$(basename "$0") pr --category Unit # Only run Unit tests
$(basename "$0") module # Auto-detect changed modules
$(basename "$0") module --module Scanner # Test specific module
$(basename "$0") workflow --workflow test-matrix
$(basename "$0") release --dry-run
$(basename "$0") pr --verbose --docker
Test Categories:
PR-Gating: ${PR_GATING_CATEGORIES[*]}
Extended: ${EXTENDED_CATEGORIES[*]}
EOF
}
# =============================================================================
# ARGUMENT PARSING
# =============================================================================
parse_args() {
while [[ $# -gt 0 ]]; do
case $1 in
smoke|pr|module|workflow|release|full)
MODE="$1"
shift
;;
--category)
SPECIFIC_CATEGORY="$2"
shift 2
;;
--workflow)
SPECIFIC_WORKFLOW="$2"
shift 2
;;
--module)
SPECIFIC_MODULE="$2"
shift 2
;;
--docker)
EXECUTION_ENGINE="docker"
shift
;;
--native)
EXECUTION_ENGINE="native"
shift
;;
--act)
EXECUTION_ENGINE="act"
shift
;;
--parallel)
PARALLEL_JOBS="$2"
shift 2
;;
--verbose|-v)
VERBOSE=true
shift
;;
--dry-run)
DRY_RUN=true
shift
;;
--rebuild)
REBUILD_IMAGE=true
shift
;;
--no-services)
SKIP_SERVICES=true
shift
;;
--keep-services)
KEEP_SERVICES=true
shift
;;
--help|-h)
usage
exit 0
;;
*)
log_error "Unknown option: $1"
usage
exit 1
;;
esac
done
# Default mode is smoke
if [[ -z "$MODE" ]]; then
MODE="$MODE_SMOKE"
fi
# Default execution engine based on mode
if [[ -z "$EXECUTION_ENGINE" ]]; then
case "$MODE" in
workflow)
EXECUTION_ENGINE="act"
;;
*)
EXECUTION_ENGINE="native"
;;
esac
fi
# Auto-detect parallel jobs
if [[ -z "$PARALLEL_JOBS" ]]; then
PARALLEL_JOBS=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
fi
export VERBOSE
}
# =============================================================================
# DEPENDENCY CHECKS
# =============================================================================
check_dependencies() {
log_subsection "Checking Dependencies"
local missing=0
# Always required
if ! require_command "dotnet" "https://dot.net/download"; then
missing=1
else
local dotnet_version
dotnet_version=$(dotnet --version 2>/dev/null || echo "unknown")
log_debug "dotnet version: $dotnet_version"
fi
if ! require_command "git"; then
missing=1
fi
# Docker required for docker mode
if [[ "$EXECUTION_ENGINE" == "docker" ]]; then
if ! check_docker; then
missing=1
fi
fi
# Act required for workflow mode
if [[ "$EXECUTION_ENGINE" == "act" ]] || [[ "$MODE" == "$MODE_WORKFLOW" ]]; then
if ! require_command "act" "brew install act (macOS) or https://github.com/nektos/act"; then
log_warn "act not found - workflow simulation will be limited"
fi
fi
# Check for solution file
if ! require_file "$REPO_ROOT/src/StellaOps.sln"; then
missing=1
fi
return $missing
}
# =============================================================================
# RESULT INITIALIZATION
# =============================================================================
init_results() {
ensure_dir "$RESULTS_DIR"
ensure_dir "$TRX_DIR"
ensure_dir "$LOGS_DIR"
# Create run metadata
local run_id
run_id=$(date +%Y%m%d_%H%M%S)
export RUN_ID="$run_id"
log_debug "Results directory: $RESULTS_DIR"
log_debug "Run ID: $RUN_ID"
}
# =============================================================================
# TEST EXECUTION
# =============================================================================
run_dotnet_tests() {
local category="$1"
local filter="Category=$category"
log_subsection "Running $category Tests"
local trx_file="$TRX_DIR/${category}-${RUN_ID}.trx"
local log_file="$LOGS_DIR/${category}-${RUN_ID}.log"
local test_cmd=(
dotnet test "$REPO_ROOT/src/StellaOps.sln"
--filter "$filter"
--configuration Release
--no-build
--logger "trx;LogFileName=$trx_file"
--results-directory "$TRX_DIR"
--verbosity minimal
)
if [[ "$DRY_RUN" == "true" ]]; then
log_info "[DRY-RUN] Would execute: ${test_cmd[*]}"
return 0
fi
local start_time
start_time=$(start_timer)
if [[ "$VERBOSE" == "true" ]]; then
"${test_cmd[@]}" 2>&1 | tee "$log_file"
else
"${test_cmd[@]}" > "$log_file" 2>&1
fi
local result=$?
stop_timer "$start_time" "$category tests"
if [[ $result -eq 0 ]]; then
log_success "$category tests passed"
else
log_error "$category tests failed (see $log_file)"
fi
return $result
}
run_dotnet_build() {
log_subsection "Building Solution"
local build_cmd=(
dotnet build "$REPO_ROOT/src/StellaOps.sln"
--configuration Release
)
if [[ "$DRY_RUN" == "true" ]]; then
log_info "[DRY-RUN] Would execute: ${build_cmd[*]}"
return 0
fi
local start_time
start_time=$(start_timer)
"${build_cmd[@]}"
local result=$?
stop_timer "$start_time" "Build"
if [[ $result -eq 0 ]]; then
log_success "Build completed successfully"
else
log_error "Build failed"
fi
return $result
}
# =============================================================================
# MODE IMPLEMENTATIONS
# =============================================================================
run_smoke_mode() {
log_section "Smoke Test Mode"
log_info "Running quick validation (Unit tests only)"
local start_time
start_time=$(start_timer)
# Build
run_dotnet_build || return 1
# Run Unit tests only
run_dotnet_tests "Unit"
local result=$?
stop_timer "$start_time" "Smoke test"
return $result
}
run_pr_mode() {
log_section "PR-Gating Mode"
log_info "Running full PR-gating suite"
log_info "Categories: ${PR_GATING_CATEGORIES[*]}"
local start_time
start_time=$(start_timer)
local failed=0
local results=()
# Check if Web module has changes
local web_changed=false
local changed_files
changed_files=$(get_changed_files main 2>/dev/null || echo "")
if echo "$changed_files" | grep -q "^src/Web/"; then
web_changed=true
log_info "Web module changes detected - will run Web tests"
fi
# Start services if needed
if [[ "$SKIP_SERVICES" != "true" ]]; then
start_ci_services postgres-ci valkey-ci || {
log_warn "Failed to start services, continuing anyway..."
}
fi
# Build .NET solution
run_dotnet_build || return 1
# Run each .NET category
if [[ -n "$SPECIFIC_CATEGORY" ]]; then
if [[ "$SPECIFIC_CATEGORY" == "Web" ]] || [[ "$SPECIFIC_CATEGORY" == "web" ]]; then
# Run Web tests only
if type run_web_pr_gating &>/dev/null; then
run_web_pr_gating
results+=("Web:$?")
fi
else
run_dotnet_tests "$SPECIFIC_CATEGORY"
results+=("$SPECIFIC_CATEGORY:$?")
fi
else
for category in "${PR_GATING_CATEGORIES[@]}"; do
run_dotnet_tests "$category"
local cat_result=$?
results+=("$category:$cat_result")
if [[ $cat_result -ne 0 ]]; then
failed=1
fi
done
# Run Web tests if Web module changed
if [[ "$web_changed" == "true" ]]; then
log_subsection "Web Module Tests"
if type run_web_pr_gating &>/dev/null; then
run_web_pr_gating
local web_result=$?
results+=("Web:$web_result")
if [[ $web_result -ne 0 ]]; then
failed=1
fi
else
log_warn "Web testing library not loaded"
fi
fi
fi
# Stop services
if [[ "$SKIP_SERVICES" != "true" ]] && [[ "$KEEP_SERVICES" != "true" ]]; then
stop_ci_services
fi
# Print summary
log_section "PR-Gating Results"
for result in "${results[@]}"; do
local name="${result%%:*}"
local status="${result##*:}"
if [[ "$status" == "0" ]]; then
print_status "$name" "true"
else
print_status "$name" "false"
fi
done
stop_timer "$start_time" "PR-gating suite"
return $failed
}
run_module_mode() {
log_section "Module-Specific Mode"
local modules_to_test=()
local has_dotnet_modules=false
local has_node_modules=false
if [[ -n "$SPECIFIC_MODULE" ]]; then
modules_to_test=("$SPECIFIC_MODULE")
log_info "Testing specified module: $SPECIFIC_MODULE"
else
log_info "Auto-detecting changed modules..."
local detected
detected=$(detect_changed_modules main)
if [[ "$detected" == "ALL" ]]; then
log_info "Infrastructure changes detected - running all tests"
run_pr_mode
return $?
elif [[ "$detected" == "NONE" ]]; then
log_info "No module changes detected"
return 0
else
read -ra modules_to_test <<< "$detected"
log_info "Detected changed modules: ${modules_to_test[*]}"
fi
fi
# Categorize modules
for module in "${modules_to_test[@]}"; do
if [[ " ${NODE_MODULES[*]} " =~ " ${module} " ]]; then
has_node_modules=true
else
has_dotnet_modules=true
fi
done
local start_time
start_time=$(start_timer)
local failed=0
# Build .NET solution if we have .NET modules
if [[ "$has_dotnet_modules" == "true" ]]; then
run_dotnet_build || return 1
fi
for module in "${modules_to_test[@]}"; do
log_subsection "Testing Module: $module"
# Check if this is a Node.js module (Web, DevPortal)
if [[ " ${NODE_MODULES[*]} " =~ " ${module} " ]]; then
log_info "Running Node.js tests for $module"
case "$module" in
Web)
if type run_web_pr_gating &>/dev/null; then
run_web_pr_gating || failed=1
else
log_warn "Web testing library not loaded - running basic npm test"
pushd "$REPO_ROOT/src/Web/StellaOps.Web" > /dev/null 2>&1 || continue
npm ci --prefer-offline --no-audit 2>/dev/null || npm install
npm run test:ci || failed=1
popd > /dev/null
fi
;;
DevPortal)
local portal_dir="$REPO_ROOT/src/DevPortal/StellaOps.DevPortal.Site"
if [[ -d "$portal_dir" ]]; then
pushd "$portal_dir" > /dev/null || continue
npm ci --prefer-offline --no-audit 2>/dev/null || npm install
npm test 2>/dev/null || log_warn "DevPortal tests not configured"
popd > /dev/null
fi
;;
esac
continue
fi
# .NET module handling
local test_paths="${MODULE_PATHS[$module]:-}"
if [[ -z "$test_paths" ]]; then
log_warn "Unknown module: $module"
continue
fi
# Run tests for each path
for path in $test_paths; do
local test_dir="$REPO_ROOT/$path/__Tests"
if [[ -d "$test_dir" ]]; then
log_info "Running tests in: $test_dir"
local test_projects
test_projects=$(find "$test_dir" -name "*.Tests.csproj" -type f 2>/dev/null)
for project in $test_projects; do
log_debug "Testing: $project"
dotnet test "$project" --configuration Release --no-build --verbosity minimal || {
failed=1
}
done
fi
done
done
stop_timer "$start_time" "Module tests"
return $failed
}
run_workflow_mode() {
log_section "Workflow Simulation Mode"
if [[ -z "$SPECIFIC_WORKFLOW" ]]; then
log_error "No workflow specified. Use --workflow <name>"
log_info "Example: --workflow test-matrix"
return 1
fi
local workflow_file="$REPO_ROOT/.gitea/workflows/${SPECIFIC_WORKFLOW}.yml"
if [[ ! -f "$workflow_file" ]]; then
# Try without .yml extension
workflow_file="$REPO_ROOT/.gitea/workflows/${SPECIFIC_WORKFLOW}"
if [[ ! -f "$workflow_file" ]]; then
log_error "Workflow not found: $SPECIFIC_WORKFLOW"
log_info "Available workflows:"
ls -1 "$REPO_ROOT/.gitea/workflows/"*.yml 2>/dev/null | xargs -n1 basename | head -20
return 1
fi
fi
log_info "Simulating workflow: $SPECIFIC_WORKFLOW"
log_info "Workflow file: $workflow_file"
if ! command -v act &>/dev/null; then
log_error "act is required for workflow simulation"
log_info "Install with: brew install act (macOS)"
return 1
fi
# Build CI image if needed
if [[ "$REBUILD_IMAGE" == "true" ]] || ! ci_image_exists; then
build_ci_image "$REBUILD_IMAGE" || return 1
fi
local event_file="$REPO_ROOT/devops/ci-local/events/pull-request.json"
local actrc_file="$REPO_ROOT/.actrc"
local act_args=(
-W "$workflow_file"
--platform "ubuntu-22.04=$CI_IMAGE"
--platform "ubuntu-latest=$CI_IMAGE"
--env "DOTNET_NOLOGO=1"
--env "DOTNET_CLI_TELEMETRY_OPTOUT=1"
--env "TZ=UTC"
--bind
)
if [[ -f "$event_file" ]]; then
act_args+=(--eventpath "$event_file")
fi
if [[ -f "$REPO_ROOT/devops/ci-local/.env.local" ]]; then
act_args+=(--env-file "$REPO_ROOT/devops/ci-local/.env.local")
fi
if [[ "$DRY_RUN" == "true" ]]; then
act_args+=(-n)
fi
if [[ "$VERBOSE" == "true" ]]; then
act_args+=(--verbose)
fi
log_info "Running: act ${act_args[*]}"
act "${act_args[@]}"
}
run_release_mode() {
log_section "Release Simulation Mode"
log_info "Running release dry-run"
if [[ "$DRY_RUN" != "true" ]]; then
log_warn "Release mode always runs as dry-run for safety"
DRY_RUN=true
fi
local start_time
start_time=$(start_timer)
# Build all modules
log_subsection "Building All Modules"
run_dotnet_build || return 1
# Package CLI
log_subsection "Packaging CLI"
local cli_project="$REPO_ROOT/src/Cli/StellaOps.Cli/StellaOps.Cli.csproj"
if [[ -f "$cli_project" ]]; then
log_info "[DRY-RUN] Would build CLI for: linux-x64, linux-arm64, osx-arm64, win-x64"
fi
# Validate Helm chart
log_subsection "Validating Helm Chart"
if command -v helm &>/dev/null; then
local helm_chart="$REPO_ROOT/devops/helm/stellaops"
if [[ -d "$helm_chart" ]]; then
helm lint "$helm_chart" || log_warn "Helm lint warnings"
fi
else
log_info "helm not found - skipping chart validation"
fi
# Generate release manifest
log_subsection "Release Manifest"
log_info "[DRY-RUN] Would generate:"
log_info " - Release notes"
log_info " - Changelog"
log_info " - Docker Compose files"
log_info " - SBOM"
log_info " - Checksums"
stop_timer "$start_time" "Release simulation"
return 0
}
run_full_mode() {
log_section "Full Test Mode"
log_info "Running all tests including extended categories"
log_info "Categories: ${ALL_CATEGORIES[*]}"
local start_time
start_time=$(start_timer)
local failed=0
# Start all services
if [[ "$SKIP_SERVICES" != "true" ]]; then
start_ci_services || {
log_warn "Failed to start services, continuing anyway..."
}
fi
# Build
run_dotnet_build || return 1
# Run all categories
for category in "${ALL_CATEGORIES[@]}"; do
run_dotnet_tests "$category" || {
failed=1
log_warn "Continuing after $category failure..."
}
done
# Stop services
if [[ "$SKIP_SERVICES" != "true" ]] && [[ "$KEEP_SERVICES" != "true" ]]; then
stop_ci_services
fi
stop_timer "$start_time" "Full test suite"
return $failed
}
# =============================================================================
# MAIN
# =============================================================================
main() {
parse_args "$@"
log_section "StellaOps Local CI Runner"
log_info "Mode: $MODE"
log_info "Engine: $EXECUTION_ENGINE"
log_info "Parallel: $PARALLEL_JOBS jobs"
log_info "Repository: $REPO_ROOT"
if [[ "$DRY_RUN" == "true" ]]; then
log_warn "DRY-RUN MODE - No changes will be made"
fi
# Check dependencies
check_dependencies || exit 1
# Initialize results directory
init_results
# Load environment
load_env_file "$REPO_ROOT/devops/ci-local/.env.local" || true
# Run selected mode
case "$MODE" in
"$MODE_SMOKE")
run_smoke_mode
;;
"$MODE_PR")
run_pr_mode
;;
"$MODE_MODULE")
run_module_mode
;;
"$MODE_WORKFLOW")
run_workflow_mode
;;
"$MODE_RELEASE")
run_release_mode
;;
"$MODE_FULL")
run_full_mode
;;
*)
log_error "Unknown mode: $MODE"
usage
exit 1
;;
esac
local result=$?
log_section "Summary"
log_info "Results saved to: $RESULTS_DIR"
if [[ $result -eq 0 ]]; then
log_success "All tests passed!"
else
log_error "Some tests failed"
fi
return $result
}
# Run main if executed directly
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
main "$@"
fi