stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

UI work to fill SBOM sourcing management gap. UI planning remaining functionality exposure. Work on CI/Tests stabilization

Browse Source 
Introduces CGS determinism test runs to CI workflows for Windows, macOS, Linux, Alpine, and Debian, fulfilling CGS-008 cross-platform requirements. Updates local-ci scripts to support new smoke steps, test timeouts, progress intervals, and project slicing for improved test isolation and diagnostics.
This commit is contained in:
master
2025-12-29 19:12:38 +02:00
parent 41552d26ec
commit a4badc275e
286 changed files with 50918 additions and 992 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
devops/scripts/local-ci.ps1
View File

@@ -25,6 +25,21 @@
.PARAMETER Workflow
Specific workflow to simulate (for workflow mode)
.PARAMETER SmokeStep
Smoke step (smoke mode only): build, unit, unit-split
.PARAMETER TestTimeout
Per-test timeout (e.g., 5m) using --blame-hang (bash runner)
.PARAMETER ProgressInterval
Progress heartbeat in seconds during long test runs
.PARAMETER ProjectStart
Start index (1-based) for unit-split slicing
.PARAMETER ProjectCount
Limit number of projects for unit-split slicing
.PARAMETER Docker
Force Docker execution mode
@@ -56,6 +71,18 @@
.\local-ci.ps1 smoke
Quick validation before push
.EXAMPLE
.\local-ci.ps1 smoke -SmokeStep unit-split
Run Unit tests per project to isolate hangs
.EXAMPLE
.\local-ci.ps1 smoke -SmokeStep unit-split -TestTimeout 5m -ProgressInterval 60
Add hang detection and progress heartbeat
.EXAMPLE
.\local-ci.ps1 smoke -SmokeStep unit-split -ProjectStart 1 -ProjectCount 50
Run unit-split in chunks to narrow the slow/hanging project
.EXAMPLE
.\local-ci.ps1 pr
Full PR check
@@ -82,14 +109,18 @@ param(
[string]$Category,
[string]$Module,
[string]$Workflow,
[ValidateSet('build', 'unit', 'unit-split')]
[string]$SmokeStep,
[string]$TestTimeout,
[int]$ProgressInterval,
[int]$ProjectStart,
[int]$ProjectCount,
[switch]$Docker,
[switch]$Native,
[switch]$Act,
[int]$Parallel,
[switch]$Verbose,
[switch]$DryRun,
[switch]$Rebuild,
[switch]$NoServices,
@@ -98,6 +129,8 @@ param(
[switch]$Help
)
$isVerbose = $PSBoundParameters.ContainsKey('Verbose')
# Script location
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
$RepoRoot = Split-Path -Parent (Split-Path -Parent $ScriptDir)
@@ -134,8 +167,15 @@ function Find-BashExecutable {
# Verify WSL is working
$wslCheck = & wsl --status 2>&1
if ($LASTEXITCODE -eq 0) {
Write-Info "Using WSL2 for Bash execution"
return @{ Type = 'wsl'; Path = 'wsl' }
$wslDotnetInfo = & wsl dotnet --info 2>&1
if ($LASTEXITCODE -eq 0 -and $wslDotnetInfo -match 'OS Name:\s+Windows') {
Write-Warning "WSL dotnet is Windows-based; falling back to Git Bash for path-safe execution"
} elseif ($LASTEXITCODE -eq 0) {
Write-Info "Using WSL2 for Bash execution"
return @{ Type = 'wsl'; Path = 'wsl' }
} else {
Write-Warning "WSL dotnet not available; falling back to Git Bash"
}
}
}
@@ -175,6 +215,13 @@ function Convert-ToUnixPath {
return $WindowsPath -replace '\\', '/'
}
function Quote-ForBash {
param([string]$Value)
$replacement = "'" + '"' + "'" + '"' + "'"
return "'" + ($Value -replace "'", $replacement) + "'"
}
# Build argument list
function Build-Arguments {
$args = @($Mode)
@@ -182,11 +229,16 @@ function Build-Arguments {
if ($Category) { $args += "--category"; $args += $Category }
if ($Module) { $args += "--module"; $args += $Module }
if ($Workflow) { $args += "--workflow"; $args += $Workflow }
if ($SmokeStep) { $args += "--smoke-step"; $args += $SmokeStep }
if ($TestTimeout) { $args += "--test-timeout"; $args += $TestTimeout }
if ($ProgressInterval) { $args += "--progress-interval"; $args += $ProgressInterval }
if ($ProjectStart) { $args += "--project-start"; $args += $ProjectStart }
if ($ProjectCount) { $args += "--project-count"; $args += $ProjectCount }
if ($Docker) { $args += "--docker" }
if ($Native) { $args += "--native" }
if ($Act) { $args += "--act" }
if ($Parallel) { $args += "--parallel"; $args += $Parallel }
if ($Verbose) { $args += "--verbose" }
if ($isVerbose) { $args += "--verbose" }
if ($DryRun) { $args += "--dry-run" }
if ($Rebuild) { $args += "--rebuild" }
if ($NoServices) { $args += "--no-services" }
@@ -237,8 +289,10 @@ try {
'gitbash' {
# Git Bash uses its own path conversion
$unixScript = $scriptPath -replace '\\', '/'
Write-Info "Executing: $($bash.Path) $unixScript $($bashArgs -join ' ')"
& $bash.Path $unixScript @bashArgs
$commandArgs = @($unixScript) + $bashArgs
$commandLine = ($commandArgs | ForEach-Object { Quote-ForBash $_ }) -join ' '
Write-Info "Executing: $($bash.Path) -lc $commandLine"
& $bash.Path -lc $commandLine
}
'path' {
Write-Info "Executing: bash $scriptPath $($bashArgs -join ' ')"

244
devops/scripts/local-ci.sh
View File

@@ -19,6 +19,11 @@
# --category <cat> Run specific test category
# --workflow <name> Specific workflow to simulate
# --module <name> Specific module to test
# --smoke-step <s> Smoke step: build, unit, unit-split (smoke mode only)
# --test-timeout <t> Per-test timeout (e.g., 5m). Adds --blame-hang timeout.
# --progress-interval <s> Progress interval in seconds for long tests
# --project-start <n> Start index (1-based) for unit-split slicing
# --project-count <n> Limit number of projects for unit-split slicing
# --docker Force Docker execution
# --native Force native execution
# --act Force act execution
@@ -75,6 +80,7 @@ ALL_CATEGORIES=("${PR_GATING_CATEGORIES[@]}" "${EXTENDED_CATEGORIES[@]}")
RESULTS_DIR="$REPO_ROOT/out/local-ci"
TRX_DIR="$RESULTS_DIR/trx"
LOGS_DIR="$RESULTS_DIR/logs"
ACTIVE_TEST_FILE="$RESULTS_DIR/active-test.txt"
# =============================================================================
# CONFIGURATION
@@ -85,6 +91,11 @@ EXECUTION_ENGINE="" # docker, native, act
SPECIFIC_CATEGORY=""
SPECIFIC_MODULE=""
SPECIFIC_WORKFLOW=""
SMOKE_STEP=""
TEST_TIMEOUT=""
PROGRESS_INTERVAL=""
PROJECT_START=""
PROJECT_COUNT=""
PARALLEL_JOBS=""
VERBOSE=false
DRY_RUN=false
@@ -112,6 +123,11 @@ Options:
--category <cat> Run specific test category (${ALL_CATEGORIES[*]})
--workflow <name> Specific workflow to simulate (for workflow mode)
--module <name> Specific module to test (for module mode)
--smoke-step <s> Smoke step (smoke mode only): build, unit, unit-split
--test-timeout <t> Per-test timeout (e.g., 5m) using --blame-hang
--progress-interval <s> Progress heartbeat in seconds
--project-start <n> Start index (1-based) for unit-split slicing
--project-count <n> Limit number of projects for unit-split slicing
--docker Force Docker execution
--native Force native execution
--act Force act execution
@@ -125,6 +141,11 @@ Options:
Examples:
$(basename "$0") smoke # Quick validation before push
$(basename "$0") smoke --smoke-step build # Build only (smoke)
$(basename "$0") smoke --smoke-step unit # Unit tests only (smoke)
$(basename "$0") smoke --smoke-step unit-split # Unit tests per project
$(basename "$0") smoke --smoke-step unit-split --test-timeout 5m --progress-interval 60
$(basename "$0") smoke --smoke-step unit-split --project-start 1 --project-count 50
$(basename "$0") pr # Full PR check
$(basename "$0") pr --category Unit # Only run Unit tests
$(basename "$0") module # Auto-detect changed modules
@@ -162,6 +183,26 @@ parse_args() {
SPECIFIC_MODULE="$2"
shift 2
;;
--smoke-step)
SMOKE_STEP="$2"
shift 2
;;
--test-timeout)
TEST_TIMEOUT="$2"
shift 2
;;
--progress-interval)
PROGRESS_INTERVAL="$2"
shift 2
;;
--project-start)
PROJECT_START="$2"
shift 2
;;
--project-count)
PROJECT_COUNT="$2"
shift 2
;;
--docker)
EXECUTION_ENGINE="docker"
shift
@@ -287,6 +328,7 @@ init_results() {
ensure_dir "$RESULTS_DIR"
ensure_dir "$TRX_DIR"
ensure_dir "$LOGS_DIR"
: > "$ACTIVE_TEST_FILE"
# Create run metadata
local run_id
@@ -310,11 +352,17 @@ run_dotnet_tests() {
local trx_file="$TRX_DIR/${category}-${RUN_ID}.trx"
local log_file="$LOGS_DIR/${category}-${RUN_ID}.log"
local blame_args=()
if [[ -n "$TEST_TIMEOUT" ]]; then
blame_args+=(--blame-hang "--blame-hang-timeout" "$TEST_TIMEOUT")
fi
local test_cmd=(
dotnet test "$REPO_ROOT/src/StellaOps.sln"
--filter "$filter"
--configuration Release
--no-build
"${blame_args[@]}"
--logger "trx;LogFileName=$trx_file"
--results-directory "$TRX_DIR"
--verbosity minimal
@@ -346,6 +394,165 @@ run_dotnet_tests() {
return $result
}
collect_test_projects() {
if command -v rg &>/dev/null; then
rg --files -g "*Tests.csproj" "$REPO_ROOT/src" | LC_ALL=C sort
else
find "$REPO_ROOT/src" -name "*Tests.csproj" -print | LC_ALL=C sort
fi
}
run_dotnet_tests_split() {
local category="$1"
local filter="Category=$category"
local progress_interval="$PROGRESS_INTERVAL"
if [[ -z "$progress_interval" ]]; then
progress_interval=60
fi
log_subsection "Running $category Tests (per project)"
local projects=()
mapfile -t projects < <(collect_test_projects)
if [[ ${#projects[@]} -eq 0 ]]; then
log_warn "No test projects found under $REPO_ROOT/src"
return 0
fi
local failed=0
local total_all="${#projects[@]}"
local start_index="${PROJECT_START:-1}"
local count_limit="${PROJECT_COUNT:-0}"
if [[ "$start_index" -lt 1 ]]; then
start_index=1
fi
if [[ "$count_limit" -lt 0 ]]; then
count_limit=0
fi
local total_to_run="$total_all"
if [[ "$count_limit" -gt 0 ]]; then
total_to_run="$count_limit"
else
total_to_run=$((total_all - start_index + 1))
if [[ "$total_to_run" -lt 0 ]]; then
total_to_run=0
fi
fi
local index=0
local run_index=0
for project in "${projects[@]}"; do
index=$((index + 1))
if [[ "$index" -lt "$start_index" ]]; then
continue
fi
if [[ "$count_limit" -gt 0 && "$run_index" -ge "$count_limit" ]]; then
break
fi
run_index=$((run_index + 1))
local project_name
project_name="$(basename "${project%.csproj}")"
log_step "$run_index" "$total_to_run" "Testing $project_name ($category)"
printf '%s %s (%s)\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$project_name" "$category" > "$ACTIVE_TEST_FILE"
local trx_file="$TRX_DIR/${category}-${RUN_ID}-${project_name}.trx"
local log_file="$LOGS_DIR/${category}-${RUN_ID}-${project_name}.log"
local blame_args=()
if [[ -n "$TEST_TIMEOUT" ]]; then
blame_args+=(--blame-hang "--blame-hang-timeout" "$TEST_TIMEOUT")
fi
local test_cmd=(
dotnet test "$project"
--filter "$filter"
--configuration Release
--no-build
"${blame_args[@]}"
--logger "trx;LogFileName=$trx_file"
--results-directory "$TRX_DIR"
--verbosity minimal
)
if [[ "$DRY_RUN" == "true" ]]; then
log_info "[DRY-RUN] Would execute: ${test_cmd[*]}"
continue
fi
local start_time
start_time=$(start_timer)
local ticker_pid=""
if [[ "$progress_interval" -gt 0 ]]; then
(
while true; do
sleep "$progress_interval"
local_now=$(get_timestamp)
local_elapsed=$((local_now - start_time))
log_info "$project_name still running after $(format_duration "$local_elapsed")"
done
) &
ticker_pid=$!
fi
set +e
if [[ "$VERBOSE" == "true" ]]; then
"${test_cmd[@]}" 2>&1 | tee "$log_file"
else
"${test_cmd[@]}" > "$log_file" 2>&1
fi
local result=$?
set -e
if [[ -n "$ticker_pid" ]]; then
kill "$ticker_pid" 2>/dev/null || true
wait "$ticker_pid" 2>/dev/null || true
fi
stop_timer "$start_time" "$project_name ($category)"
if [[ $result -ne 0 ]] && grep -q -E "The test source file .* was not found" "$log_file"; then
log_warn "$project_name output missing; retrying with build"
local retry_cmd=(
dotnet test "$project"
--filter "$filter"
--configuration Release
"${blame_args[@]}"
--logger "trx;LogFileName=$trx_file"
--results-directory "$TRX_DIR"
--verbosity minimal
)
local retry_start
retry_start=$(start_timer)
set +e
if [[ "$VERBOSE" == "true" ]]; then
"${retry_cmd[@]}" 2>&1 | tee -a "$log_file"
else
"${retry_cmd[@]}" >> "$log_file" 2>&1
fi
result=$?
set -e
stop_timer "$retry_start" "$project_name ($category) rebuild"
fi
if [[ $result -eq 0 ]]; then
log_success "$project_name $category tests passed"
else
if grep -q -E "No test matches the given testcase filter|No test is available" "$log_file"; then
log_warn "$project_name has no $category tests; skipping"
else
log_error "$project_name $category tests failed (see $log_file)"
failed=1
fi
fi
done
return $failed
}
run_dotnet_build() {
log_subsection "Building Solution"
@@ -382,17 +589,42 @@ run_dotnet_build() {
run_smoke_mode() {
log_section "Smoke Test Mode"
log_info "Running quick validation (Unit tests only)"
if [[ -n "$SMOKE_STEP" ]]; then
log_info "Running smoke step: $SMOKE_STEP"
else
log_info "Running quick validation (Unit tests only)"
fi
local start_time
start_time=$(start_timer)
# Build
run_dotnet_build || return 1
local result=0
case "$SMOKE_STEP" in
"" )
# Build
run_dotnet_build || return 1
# Run Unit tests only
run_dotnet_tests "Unit"
local result=$?
# Run Unit tests only
run_dotnet_tests "Unit"
result=$?
;;
build )
run_dotnet_build
result=$?
;;
unit )
run_dotnet_tests "Unit"
result=$?
;;
unit-split )
run_dotnet_tests_split "Unit"
result=$?
;;
* )
log_error "Unknown smoke step: $SMOKE_STEP"
return 1
;;
esac
stop_timer "$start_time" "Smoke test"
return $result