stabilize tests
This commit is contained in:
master
372
scripts/test-stabilization/run-tests-batch.ps1
Normal file
372
scripts/test-stabilization/run-tests-batch.ps1
Normal file
@@ -0,0 +1,372 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Runs .NET test projects in batches with timeout handling and binary search for hanging tests.
|
||||
|
||||
.DESCRIPTION
|
||||
This script:
|
||||
1. Discovers all test projects (excluding EvidenceLocker.Tests)
|
||||
2. Runs tests in batches of configurable size (default 50)
|
||||
3. Implements 50-minute timeout per batch
|
||||
4. Uses binary search to identify hanging test projects when timeout occurs
|
||||
5. Logs results to CSV and detailed logs
|
||||
|
||||
.PARAMETER BatchSize
|
||||
Number of projects per batch. Default: 50
|
||||
|
||||
.PARAMETER TimeoutMinutes
|
||||
Timeout in minutes per batch. Default: 50
|
||||
|
||||
.PARAMETER OutputDir
|
||||
Directory for output files. Default: ./test-results
|
||||
|
||||
.PARAMETER StartBatch
|
||||
Batch number to start from (0-indexed). Default: 0
|
||||
|
||||
.PARAMETER ProjectList
|
||||
Optional path to a file containing project paths (one per line)
|
||||
#>
|
||||
|
||||
param(
|
||||
[int]$BatchSize = 50,
|
||||
[int]$TimeoutMinutes = 50,
|
||||
[string]$OutputDir = "./test-results",
|
||||
[int]$StartBatch = 0,
|
||||
[string]$ProjectList = ""
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Continue"
|
||||
$RepoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot)
|
||||
|
||||
# Ensure output directory exists
|
||||
$OutputDir = Join-Path $RepoRoot $OutputDir
|
||||
if (-not (Test-Path $OutputDir)) {
|
||||
New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null
|
||||
}
|
||||
|
||||
$Timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||||
$ResultsFile = Join-Path $OutputDir "test-results-$Timestamp.csv"
|
||||
$LogFile = Join-Path $OutputDir "test-log-$Timestamp.txt"
|
||||
$HangingProjectsFile = Join-Path $OutputDir "hanging-projects-$Timestamp.txt"
|
||||
$TimeoutProjectsFile = Join-Path $OutputDir "timeout-projects.txt"
|
||||
|
||||
function Write-Log {
|
||||
param([string]$Message)
|
||||
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
|
||||
$logMessage = "[$timestamp] $Message"
|
||||
Write-Host $logMessage
|
||||
Add-Content -Path $LogFile -Value $logMessage
|
||||
}
|
||||
|
||||
function Get-TestProjects {
|
||||
Write-Log "Discovering test projects..."
|
||||
|
||||
if ($ProjectList -and (Test-Path $ProjectList)) {
|
||||
$projects = Get-Content $ProjectList | Where-Object { $_ -match "\.csproj$" }
|
||||
} else {
|
||||
$projects = Get-ChildItem -Path (Join-Path $RepoRoot "src") -Recurse -Filter "*.csproj" |
|
||||
Where-Object {
|
||||
$_.Name -match "\.Tests\.csproj$" -or
|
||||
$_.FullName -match "__Tests"
|
||||
} |
|
||||
Where-Object {
|
||||
# Exclude fixture projects and bin/obj directories
|
||||
$_.FullName -notmatch "\\bin\\" -and
|
||||
$_.FullName -notmatch "\\obj\\" -and
|
||||
$_.FullName -notmatch "\\Fixtures\\" -and
|
||||
$_.Name -notmatch "Sample\.App\.csproj" -and
|
||||
# Exclude EvidenceLocker.Tests (requires 256GB RAM)
|
||||
$_.Name -ne "StellaOps.EvidenceLocker.Tests.csproj"
|
||||
} |
|
||||
ForEach-Object { $_.FullName }
|
||||
}
|
||||
|
||||
Write-Log "Found $($projects.Count) test projects"
|
||||
return $projects
|
||||
}
|
||||
|
||||
function Run-SingleTestProject {
|
||||
param(
|
||||
[string]$ProjectPath,
|
||||
[int]$TimeoutSeconds = 300
|
||||
)
|
||||
|
||||
$projectName = [System.IO.Path]::GetFileNameWithoutExtension($ProjectPath)
|
||||
$result = @{
|
||||
Project = $projectName
|
||||
Path = $ProjectPath
|
||||
Status = "Unknown"
|
||||
Errors = 0
|
||||
Warnings = 0
|
||||
Total = 0
|
||||
Passed = 0
|
||||
Failed = 0
|
||||
Skipped = 0
|
||||
Duration = 0
|
||||
Message = ""
|
||||
}
|
||||
|
||||
$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
|
||||
try {
|
||||
$logOutput = Join-Path $OutputDir "logs"
|
||||
if (-not (Test-Path $logOutput)) {
|
||||
New-Item -ItemType Directory -Path $logOutput -Force | Out-Null
|
||||
}
|
||||
$projectLog = Join-Path $logOutput "$projectName.log"
|
||||
|
||||
$process = Start-Process -FilePath "dotnet" `
|
||||
-ArgumentList "test", "`"$ProjectPath`"", "--no-build", "--logger", "trx", "--verbosity", "minimal" `
|
||||
-NoNewWindow -PassThru -RedirectStandardOutput $projectLog -RedirectStandardError "$projectLog.err"
|
||||
|
||||
$completed = $process.WaitForExit($TimeoutSeconds * 1000)
|
||||
$stopwatch.Stop()
|
||||
$result.Duration = [math]::Round($stopwatch.Elapsed.TotalSeconds, 2)
|
||||
|
||||
if (-not $completed) {
|
||||
$process.Kill()
|
||||
$result.Status = "Timeout"
|
||||
$result.Message = "Test timed out after $TimeoutSeconds seconds"
|
||||
return $result
|
||||
}
|
||||
|
||||
$exitCode = $process.ExitCode
|
||||
|
||||
if (Test-Path $projectLog) {
|
||||
$output = Get-Content $projectLog -Raw -ErrorAction SilentlyContinue
|
||||
|
||||
# Parse test results from output
|
||||
if ($output -match "Passed:\s*(\d+)") { $result.Passed = [int]$Matches[1] }
|
||||
if ($output -match "Failed:\s*(\d+)") { $result.Failed = [int]$Matches[1] }
|
||||
if ($output -match "Skipped:\s*(\d+)") { $result.Skipped = [int]$Matches[1] }
|
||||
if ($output -match "Total:\s*(\d+)") { $result.Total = [int]$Matches[1] }
|
||||
|
||||
# Count errors and warnings
|
||||
$result.Errors = ([regex]::Matches($output, "error [A-Z]+\d+:")).Count
|
||||
$result.Warnings = ([regex]::Matches($output, "warning [A-Z]+\d+:")).Count
|
||||
}
|
||||
|
||||
if ($exitCode -eq 0) {
|
||||
$result.Status = "Passed"
|
||||
} elseif ($result.Errors -gt 0) {
|
||||
$result.Status = "BuildError"
|
||||
} elseif ($result.Failed -gt 0) {
|
||||
$result.Status = "Failed"
|
||||
} else {
|
||||
$result.Status = "Error"
|
||||
$result.Message = "Exit code: $exitCode"
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$stopwatch.Stop()
|
||||
$result.Status = "Exception"
|
||||
$result.Message = $_.Exception.Message
|
||||
$result.Duration = [math]::Round($stopwatch.Elapsed.TotalSeconds, 2)
|
||||
}
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
function Run-BatchWithTimeout {
|
||||
param(
|
||||
[string[]]$Projects,
|
||||
[int]$BatchNum,
|
||||
[int]$TimeoutMinutes
|
||||
)
|
||||
|
||||
Write-Log "Starting batch $BatchNum with $($Projects.Count) projects (timeout: $TimeoutMinutes minutes)"
|
||||
|
||||
$batchResults = @()
|
||||
$batchStopwatch = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$timeoutSeconds = $TimeoutMinutes * 60
|
||||
$perProjectTimeout = [math]::Min(300, [math]::Floor($timeoutSeconds / $Projects.Count))
|
||||
|
||||
foreach ($project in $Projects) {
|
||||
$projectName = [System.IO.Path]::GetFileNameWithoutExtension($project)
|
||||
|
||||
# Check if batch timeout exceeded
|
||||
if ($batchStopwatch.Elapsed.TotalSeconds -gt $timeoutSeconds) {
|
||||
Write-Log "BATCH TIMEOUT: Batch $BatchNum exceeded $TimeoutMinutes minutes"
|
||||
return @{
|
||||
Results = $batchResults
|
||||
TimedOut = $true
|
||||
RemainingProjects = $Projects | Where-Object { $batchResults.Path -notcontains $_ }
|
||||
}
|
||||
}
|
||||
|
||||
Write-Log " Testing: $projectName"
|
||||
$result = Run-SingleTestProject -ProjectPath $project -TimeoutSeconds $perProjectTimeout
|
||||
$batchResults += $result
|
||||
|
||||
$statusIcon = switch ($result.Status) {
|
||||
"Passed" { "[OK]" }
|
||||
"Failed" { "[FAIL]" }
|
||||
"BuildError" { "[BUILD]" }
|
||||
"Timeout" { "[TIMEOUT]" }
|
||||
default { "[?]" }
|
||||
}
|
||||
|
||||
Write-Log " $statusIcon $($result.Status) - $($result.Passed)/$($result.Total) passed, $($result.Duration)s"
|
||||
}
|
||||
|
||||
$batchStopwatch.Stop()
|
||||
Write-Log "Batch $BatchNum completed in $([math]::Round($batchStopwatch.Elapsed.TotalMinutes, 2)) minutes"
|
||||
|
||||
return @{
|
||||
Results = $batchResults
|
||||
TimedOut = $false
|
||||
RemainingProjects = @()
|
||||
}
|
||||
}
|
||||
|
||||
function Binary-SearchHangingProject {
|
||||
param(
|
||||
[string[]]$Projects,
|
||||
[int]$TimeoutMinutes
|
||||
)
|
||||
|
||||
Write-Log "BINARY SEARCH: Starting binary search for hanging project in $($Projects.Count) projects"
|
||||
|
||||
if ($Projects.Count -eq 1) {
|
||||
Write-Log "BINARY SEARCH: Found hanging project: $($Projects[0])"
|
||||
Add-Content -Path $HangingProjectsFile -Value $Projects[0]
|
||||
return $Projects[0]
|
||||
}
|
||||
|
||||
$mid = [math]::Floor($Projects.Count / 2)
|
||||
$firstHalf = $Projects[0..($mid-1)]
|
||||
$secondHalf = $Projects[$mid..($Projects.Count-1)]
|
||||
|
||||
Write-Log "BINARY SEARCH: Testing first half ($($firstHalf.Count) projects)"
|
||||
$reducedTimeout = [math]::Max(5, [math]::Floor($TimeoutMinutes / 2))
|
||||
|
||||
$firstResult = Run-BatchWithTimeout -Projects $firstHalf -BatchNum -1 -TimeoutMinutes $reducedTimeout
|
||||
|
||||
if ($firstResult.TimedOut) {
|
||||
Write-Log "BINARY SEARCH: Timeout in first half, searching deeper"
|
||||
return Binary-SearchHangingProject -Projects $firstHalf -TimeoutMinutes $reducedTimeout
|
||||
}
|
||||
|
||||
Write-Log "BINARY SEARCH: First half completed, testing second half ($($secondHalf.Count) projects)"
|
||||
$secondResult = Run-BatchWithTimeout -Projects $secondHalf -BatchNum -1 -TimeoutMinutes $reducedTimeout
|
||||
|
||||
if ($secondResult.TimedOut) {
|
||||
Write-Log "BINARY SEARCH: Timeout in second half, searching deeper"
|
||||
return Binary-SearchHangingProject -Projects $secondHalf -TimeoutMinutes $reducedTimeout
|
||||
}
|
||||
|
||||
Write-Log "BINARY SEARCH: No timeout found in either half (possible intermittent issue)"
|
||||
return $null
|
||||
}
|
||||
|
||||
# Initialize CSV with headers
|
||||
"Project,Path,Status,Errors,Warnings,Total,Passed,Failed,Skipped,Duration,Message" | Out-File -FilePath $ResultsFile -Encoding UTF8
|
||||
|
||||
Write-Log "=========================================="
|
||||
Write-Log "Test Stabilization Run"
|
||||
Write-Log "Batch Size: $BatchSize"
|
||||
Write-Log "Timeout: $TimeoutMinutes minutes per batch"
|
||||
Write-Log "Output: $OutputDir"
|
||||
Write-Log "=========================================="
|
||||
|
||||
# Get all test projects
|
||||
$allProjects = Get-TestProjects
|
||||
|
||||
# Load previously identified timeout projects to skip
|
||||
$skipProjects = @()
|
||||
if (Test-Path $TimeoutProjectsFile) {
|
||||
$skipProjects = Get-Content $TimeoutProjectsFile
|
||||
Write-Log "Loaded $($skipProjects.Count) known timeout projects to skip"
|
||||
}
|
||||
|
||||
$allProjects = $allProjects | Where-Object { $skipProjects -notcontains $_ }
|
||||
Write-Log "Running $($allProjects.Count) projects after exclusions"
|
||||
|
||||
# Split into batches
|
||||
$batches = @()
|
||||
for ($i = 0; $i -lt $allProjects.Count; $i += $BatchSize) {
|
||||
$end = [math]::Min($i + $BatchSize - 1, $allProjects.Count - 1)
|
||||
$batches += ,($allProjects[$i..$end])
|
||||
}
|
||||
|
||||
Write-Log "Created $($batches.Count) batches"
|
||||
|
||||
# First, build all projects
|
||||
Write-Log "Building solution..."
|
||||
$buildProcess = Start-Process -FilePath "dotnet" `
|
||||
-ArgumentList "build", (Join-Path $RepoRoot "src"), "--configuration", "Release", "--verbosity", "minimal" `
|
||||
-NoNewWindow -PassThru -Wait
|
||||
|
||||
if ($buildProcess.ExitCode -ne 0) {
|
||||
Write-Log "WARNING: Solution build had errors, continuing with test execution"
|
||||
}
|
||||
|
||||
# Run batches
|
||||
$allResults = @()
|
||||
$totalStats = @{
|
||||
Passed = 0
|
||||
Failed = 0
|
||||
BuildError = 0
|
||||
Timeout = 0
|
||||
Total = 0
|
||||
}
|
||||
|
||||
for ($batchNum = $StartBatch; $batchNum -lt $batches.Count; $batchNum++) {
|
||||
$batch = $batches[$batchNum]
|
||||
Write-Log ""
|
||||
Write-Log "=========================================="
|
||||
Write-Log "BATCH $($batchNum + 1) of $($batches.Count)"
|
||||
Write-Log "=========================================="
|
||||
|
||||
$batchResult = Run-BatchWithTimeout -Projects $batch -BatchNum $batchNum -TimeoutMinutes $TimeoutMinutes
|
||||
|
||||
if ($batchResult.TimedOut) {
|
||||
Write-Log "Batch $batchNum timed out, initiating binary search..."
|
||||
$hangingProject = Binary-SearchHangingProject -Projects $batchResult.RemainingProjects -TimeoutMinutes $TimeoutMinutes
|
||||
|
||||
if ($hangingProject) {
|
||||
Write-Log "Adding $hangingProject to timeout projects list"
|
||||
Add-Content -Path $TimeoutProjectsFile -Value $hangingProject
|
||||
}
|
||||
}
|
||||
|
||||
# Record results
|
||||
foreach ($result in $batchResult.Results) {
|
||||
$csvLine = "$($result.Project),$($result.Path),$($result.Status),$($result.Errors),$($result.Warnings),$($result.Total),$($result.Passed),$($result.Failed),$($result.Skipped),$($result.Duration),`"$($result.Message)`""
|
||||
Add-Content -Path $ResultsFile -Value $csvLine
|
||||
|
||||
$totalStats.Total++
|
||||
switch ($result.Status) {
|
||||
"Passed" { $totalStats.Passed++ }
|
||||
"Failed" { $totalStats.Failed++ }
|
||||
"BuildError" { $totalStats.BuildError++ }
|
||||
"Timeout" { $totalStats.Timeout++ }
|
||||
}
|
||||
}
|
||||
|
||||
$allResults += $batchResult.Results
|
||||
|
||||
# Progress summary
|
||||
Write-Log ""
|
||||
Write-Log "Progress: $($totalStats.Total) projects tested"
|
||||
Write-Log " Passed: $($totalStats.Passed)"
|
||||
Write-Log " Failed: $($totalStats.Failed)"
|
||||
Write-Log " Build Errors: $($totalStats.BuildError)"
|
||||
Write-Log " Timeouts: $($totalStats.Timeout)"
|
||||
}
|
||||
|
||||
Write-Log ""
|
||||
Write-Log "=========================================="
|
||||
Write-Log "FINAL SUMMARY"
|
||||
Write-Log "=========================================="
|
||||
Write-Log "Total Projects: $($totalStats.Total)"
|
||||
Write-Log "Passed: $($totalStats.Passed) ($([math]::Round($totalStats.Passed / [math]::Max(1, $totalStats.Total) * 100, 1))%)"
|
||||
Write-Log "Failed: $($totalStats.Failed)"
|
||||
Write-Log "Build Errors: $($totalStats.BuildError)"
|
||||
Write-Log "Timeouts: $($totalStats.Timeout)"
|
||||
Write-Log ""
|
||||
Write-Log "Results saved to: $ResultsFile"
|
||||
Write-Log "Log saved to: $LogFile"
|
||||
|
||||
if (Test-Path $HangingProjectsFile) {
|
||||
Write-Log "Hanging projects saved to: $HangingProjectsFile"
|
||||
}
|
||||
Reference in New Issue
Block a user