<# .SYNOPSIS Runs .NET test projects in batches with timeout handling and binary search for hanging tests. .DESCRIPTION This script: 1. Discovers all test projects (excluding EvidenceLocker.Tests) 2. Runs tests in batches of configurable size (default 50) 3. Implements 50-minute timeout per batch 4. Uses binary search to identify hanging test projects when timeout occurs 5. Logs results to CSV and detailed logs .PARAMETER BatchSize Number of projects per batch. Default: 50 .PARAMETER TimeoutMinutes Timeout in minutes per batch. Default: 50 .PARAMETER OutputDir Directory for output files. Default: ./test-results .PARAMETER StartBatch Batch number to start from (0-indexed). Default: 0 .PARAMETER ProjectList Optional path to a file containing project paths (one per line) #> param( [int]$BatchSize = 50, [int]$TimeoutMinutes = 50, [string]$OutputDir = "./test-results", [int]$StartBatch = 0, [string]$ProjectList = "" ) $ErrorActionPreference = "Continue" $RepoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) # Ensure output directory exists $OutputDir = Join-Path $RepoRoot $OutputDir if (-not (Test-Path $OutputDir)) { New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null } $Timestamp = Get-Date -Format "yyyyMMdd_HHmmss" $ResultsFile = Join-Path $OutputDir "test-results-$Timestamp.csv" $LogFile = Join-Path $OutputDir "test-log-$Timestamp.txt" $HangingProjectsFile = Join-Path $OutputDir "hanging-projects-$Timestamp.txt" $TimeoutProjectsFile = Join-Path $OutputDir "timeout-projects.txt" function Write-Log { param([string]$Message) $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" $logMessage = "[$timestamp] $Message" Write-Host $logMessage Add-Content -Path $LogFile -Value $logMessage } function Get-TestProjects { Write-Log "Discovering test projects..." if ($ProjectList -and (Test-Path $ProjectList)) { $projects = Get-Content $ProjectList | Where-Object { $_ -match "\.csproj$" } } else { $projects = Get-ChildItem -Path (Join-Path $RepoRoot "src") -Recurse -Filter "*.csproj" | Where-Object { $_.Name -match "\.Tests\.csproj$" -or $_.FullName -match "__Tests" } | Where-Object { # Exclude fixture projects and bin/obj directories $_.FullName -notmatch "\\bin\\" -and $_.FullName -notmatch "\\obj\\" -and $_.FullName -notmatch "\\Fixtures\\" -and $_.Name -notmatch "Sample\.App\.csproj" -and # Exclude EvidenceLocker.Tests (requires 256GB RAM) $_.Name -ne "StellaOps.EvidenceLocker.Tests.csproj" } | ForEach-Object { $_.FullName } } Write-Log "Found $($projects.Count) test projects" return $projects } function Run-SingleTestProject { param( [string]$ProjectPath, [int]$TimeoutSeconds = 300 ) $projectName = [System.IO.Path]::GetFileNameWithoutExtension($ProjectPath) $result = @{ Project = $projectName Path = $ProjectPath Status = "Unknown" Errors = 0 Warnings = 0 Total = 0 Passed = 0 Failed = 0 Skipped = 0 Duration = 0 Message = "" } $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() try { $logOutput = Join-Path $OutputDir "logs" if (-not (Test-Path $logOutput)) { New-Item -ItemType Directory -Path $logOutput -Force | Out-Null } $projectLog = Join-Path $logOutput "$projectName.log" $process = Start-Process -FilePath "dotnet" ` -ArgumentList "test", "`"$ProjectPath`"", "--no-build", "--logger", "trx", "--verbosity", "minimal" ` -NoNewWindow -PassThru -RedirectStandardOutput $projectLog -RedirectStandardError "$projectLog.err" $completed = $process.WaitForExit($TimeoutSeconds * 1000) $stopwatch.Stop() $result.Duration = [math]::Round($stopwatch.Elapsed.TotalSeconds, 2) if (-not $completed) { $process.Kill() $result.Status = "Timeout" $result.Message = "Test timed out after $TimeoutSeconds seconds" return $result } $exitCode = $process.ExitCode if (Test-Path $projectLog) { $output = Get-Content $projectLog -Raw -ErrorAction SilentlyContinue # Parse test results from output if ($output -match "Passed:\s*(\d+)") { $result.Passed = [int]$Matches[1] } if ($output -match "Failed:\s*(\d+)") { $result.Failed = [int]$Matches[1] } if ($output -match "Skipped:\s*(\d+)") { $result.Skipped = [int]$Matches[1] } if ($output -match "Total:\s*(\d+)") { $result.Total = [int]$Matches[1] } # Count errors and warnings $result.Errors = ([regex]::Matches($output, "error [A-Z]+\d+:")).Count $result.Warnings = ([regex]::Matches($output, "warning [A-Z]+\d+:")).Count } if ($exitCode -eq 0) { $result.Status = "Passed" } elseif ($result.Errors -gt 0) { $result.Status = "BuildError" } elseif ($result.Failed -gt 0) { $result.Status = "Failed" } else { $result.Status = "Error" $result.Message = "Exit code: $exitCode" } } catch { $stopwatch.Stop() $result.Status = "Exception" $result.Message = $_.Exception.Message $result.Duration = [math]::Round($stopwatch.Elapsed.TotalSeconds, 2) } return $result } function Run-BatchWithTimeout { param( [string[]]$Projects, [int]$BatchNum, [int]$TimeoutMinutes ) Write-Log "Starting batch $BatchNum with $($Projects.Count) projects (timeout: $TimeoutMinutes minutes)" $batchResults = @() $batchStopwatch = [System.Diagnostics.Stopwatch]::StartNew() $timeoutSeconds = $TimeoutMinutes * 60 $perProjectTimeout = [math]::Min(300, [math]::Floor($timeoutSeconds / $Projects.Count)) foreach ($project in $Projects) { $projectName = [System.IO.Path]::GetFileNameWithoutExtension($project) # Check if batch timeout exceeded if ($batchStopwatch.Elapsed.TotalSeconds -gt $timeoutSeconds) { Write-Log "BATCH TIMEOUT: Batch $BatchNum exceeded $TimeoutMinutes minutes" return @{ Results = $batchResults TimedOut = $true RemainingProjects = $Projects | Where-Object { $batchResults.Path -notcontains $_ } } } Write-Log " Testing: $projectName" $result = Run-SingleTestProject -ProjectPath $project -TimeoutSeconds $perProjectTimeout $batchResults += $result $statusIcon = switch ($result.Status) { "Passed" { "[OK]" } "Failed" { "[FAIL]" } "BuildError" { "[BUILD]" } "Timeout" { "[TIMEOUT]" } default { "[?]" } } Write-Log " $statusIcon $($result.Status) - $($result.Passed)/$($result.Total) passed, $($result.Duration)s" } $batchStopwatch.Stop() Write-Log "Batch $BatchNum completed in $([math]::Round($batchStopwatch.Elapsed.TotalMinutes, 2)) minutes" return @{ Results = $batchResults TimedOut = $false RemainingProjects = @() } } function Binary-SearchHangingProject { param( [string[]]$Projects, [int]$TimeoutMinutes ) Write-Log "BINARY SEARCH: Starting binary search for hanging project in $($Projects.Count) projects" if ($Projects.Count -eq 1) { Write-Log "BINARY SEARCH: Found hanging project: $($Projects[0])" Add-Content -Path $HangingProjectsFile -Value $Projects[0] return $Projects[0] } $mid = [math]::Floor($Projects.Count / 2) $firstHalf = $Projects[0..($mid-1)] $secondHalf = $Projects[$mid..($Projects.Count-1)] Write-Log "BINARY SEARCH: Testing first half ($($firstHalf.Count) projects)" $reducedTimeout = [math]::Max(5, [math]::Floor($TimeoutMinutes / 2)) $firstResult = Run-BatchWithTimeout -Projects $firstHalf -BatchNum -1 -TimeoutMinutes $reducedTimeout if ($firstResult.TimedOut) { Write-Log "BINARY SEARCH: Timeout in first half, searching deeper" return Binary-SearchHangingProject -Projects $firstHalf -TimeoutMinutes $reducedTimeout } Write-Log "BINARY SEARCH: First half completed, testing second half ($($secondHalf.Count) projects)" $secondResult = Run-BatchWithTimeout -Projects $secondHalf -BatchNum -1 -TimeoutMinutes $reducedTimeout if ($secondResult.TimedOut) { Write-Log "BINARY SEARCH: Timeout in second half, searching deeper" return Binary-SearchHangingProject -Projects $secondHalf -TimeoutMinutes $reducedTimeout } Write-Log "BINARY SEARCH: No timeout found in either half (possible intermittent issue)" return $null } # Initialize CSV with headers "Project,Path,Status,Errors,Warnings,Total,Passed,Failed,Skipped,Duration,Message" | Out-File -FilePath $ResultsFile -Encoding UTF8 Write-Log "==========================================" Write-Log "Test Stabilization Run" Write-Log "Batch Size: $BatchSize" Write-Log "Timeout: $TimeoutMinutes minutes per batch" Write-Log "Output: $OutputDir" Write-Log "==========================================" # Get all test projects $allProjects = Get-TestProjects # Load previously identified timeout projects to skip $skipProjects = @() if (Test-Path $TimeoutProjectsFile) { $skipProjects = Get-Content $TimeoutProjectsFile Write-Log "Loaded $($skipProjects.Count) known timeout projects to skip" } $allProjects = $allProjects | Where-Object { $skipProjects -notcontains $_ } Write-Log "Running $($allProjects.Count) projects after exclusions" # Split into batches $batches = @() for ($i = 0; $i -lt $allProjects.Count; $i += $BatchSize) { $end = [math]::Min($i + $BatchSize - 1, $allProjects.Count - 1) $batches += ,($allProjects[$i..$end]) } Write-Log "Created $($batches.Count) batches" # First, build all projects Write-Log "Building solution..." $buildProcess = Start-Process -FilePath "dotnet" ` -ArgumentList "build", (Join-Path $RepoRoot "src"), "--configuration", "Release", "--verbosity", "minimal" ` -NoNewWindow -PassThru -Wait if ($buildProcess.ExitCode -ne 0) { Write-Log "WARNING: Solution build had errors, continuing with test execution" } # Run batches $allResults = @() $totalStats = @{ Passed = 0 Failed = 0 BuildError = 0 Timeout = 0 Total = 0 } for ($batchNum = $StartBatch; $batchNum -lt $batches.Count; $batchNum++) { $batch = $batches[$batchNum] Write-Log "" Write-Log "==========================================" Write-Log "BATCH $($batchNum + 1) of $($batches.Count)" Write-Log "==========================================" $batchResult = Run-BatchWithTimeout -Projects $batch -BatchNum $batchNum -TimeoutMinutes $TimeoutMinutes if ($batchResult.TimedOut) { Write-Log "Batch $batchNum timed out, initiating binary search..." $hangingProject = Binary-SearchHangingProject -Projects $batchResult.RemainingProjects -TimeoutMinutes $TimeoutMinutes if ($hangingProject) { Write-Log "Adding $hangingProject to timeout projects list" Add-Content -Path $TimeoutProjectsFile -Value $hangingProject } } # Record results foreach ($result in $batchResult.Results) { $csvLine = "$($result.Project),$($result.Path),$($result.Status),$($result.Errors),$($result.Warnings),$($result.Total),$($result.Passed),$($result.Failed),$($result.Skipped),$($result.Duration),`"$($result.Message)`"" Add-Content -Path $ResultsFile -Value $csvLine $totalStats.Total++ switch ($result.Status) { "Passed" { $totalStats.Passed++ } "Failed" { $totalStats.Failed++ } "BuildError" { $totalStats.BuildError++ } "Timeout" { $totalStats.Timeout++ } } } $allResults += $batchResult.Results # Progress summary Write-Log "" Write-Log "Progress: $($totalStats.Total) projects tested" Write-Log " Passed: $($totalStats.Passed)" Write-Log " Failed: $($totalStats.Failed)" Write-Log " Build Errors: $($totalStats.BuildError)" Write-Log " Timeouts: $($totalStats.Timeout)" } Write-Log "" Write-Log "==========================================" Write-Log "FINAL SUMMARY" Write-Log "==========================================" Write-Log "Total Projects: $($totalStats.Total)" Write-Log "Passed: $($totalStats.Passed) ($([math]::Round($totalStats.Passed / [math]::Max(1, $totalStats.Total) * 100, 1))%)" Write-Log "Failed: $($totalStats.Failed)" Write-Log "Build Errors: $($totalStats.BuildError)" Write-Log "Timeouts: $($totalStats.Timeout)" Write-Log "" Write-Log "Results saved to: $ResultsFile" Write-Log "Log saved to: $LogFile" if (Test-Path $HangingProjectsFile) { Write-Log "Hanging projects saved to: $HangingProjectsFile" }