Files
git.stella-ops.org/devops/compose/scripts/router-mode-redeploy.ps1

178 lines
5.4 KiB
PowerShell

param(
[ValidateSet("microservice", "reverseproxy")]
[string]$Mode = "microservice",
[string]$ComposeFile = "docker-compose.stella-ops.yml",
[int]$WaitTimeoutSeconds = 1200,
[int]$RecoveryAttempts = 2,
[int]$RecoveryWaitSeconds = 180,
[switch]$SkipHeaderSearchSmoke
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
$composeDirectory = Split-Path -Parent $PSScriptRoot
$resolvedComposeFile = if ([System.IO.Path]::IsPathRooted($ComposeFile)) {
$ComposeFile
} else {
Join-Path -Path $composeDirectory -ChildPath $ComposeFile
}
if (-not (Test-Path -LiteralPath $resolvedComposeFile)) {
throw "Compose file not found: $resolvedComposeFile"
}
$configFileName = switch ($Mode) {
"microservice" { "router-gateway-local.json" }
"reverseproxy" { "router-gateway-local.reverseproxy.json" }
default { throw "Unsupported mode: $Mode" }
}
$configPath = Join-Path -Path $composeDirectory -ChildPath $configFileName
if (-not (Test-Path -LiteralPath $configPath)) {
throw "Gateway config file not found: $configPath"
}
Write-Host "Redeploy mode: $Mode"
Write-Host "Gateway config: $configPath"
Write-Host "Compose file: $resolvedComposeFile"
$env:ROUTER_GATEWAY_CONFIG = "./$configFileName"
function Invoke-Compose {
param(
[Parameter(Mandatory = $true)]
[string[]]$Args,
[switch]$IgnoreExitCode
)
& docker compose --project-directory $composeDirectory -f $resolvedComposeFile @Args
$exitCode = $LASTEXITCODE
if (-not $IgnoreExitCode -and $exitCode -ne 0) {
throw "docker compose $($Args -join ' ') failed with exit code $exitCode."
}
return $exitCode
}
function Get-UnhealthyContainers {
$containers = & docker ps --filter "health=unhealthy" --format "{{.Names}}"
if ($LASTEXITCODE -ne 0) {
throw "Failed to query unhealthy containers."
}
$filtered = @($containers | Where-Object { -not [string]::IsNullOrWhiteSpace($_) -and $_ -like "stellaops-*" })
return [string[]]$filtered
}
function Get-ComposeServiceName {
param(
[Parameter(Mandatory = $true)]
[string]$ContainerName
)
$inspectJson = & docker inspect $ContainerName 2>$null
if ($LASTEXITCODE -ne 0 -or $null -eq $inspectJson) {
return $null
}
try {
$inspect = $inspectJson | ConvertFrom-Json
if ($null -eq $inspect) {
return $null
}
$first = if ($inspect -is [System.Array]) { $inspect[0] } else { $inspect }
$labels = $first.Config.Labels
if ($null -eq $labels) {
return $null
}
$service = $labels."com.docker.compose.service"
if ([string]::IsNullOrWhiteSpace($service)) {
return $null
}
return $service.Trim()
}
catch {
return $null
}
}
function Wait-ForContainerHealth {
param(
[Parameter(Mandatory = $true)]
[string]$ContainerName,
[Parameter(Mandatory = $true)]
[int]$TimeoutSeconds
)
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
while ((Get-Date) -lt $deadline) {
$status = (& docker inspect --format "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}" $ContainerName 2>$null).Trim()
if ($LASTEXITCODE -ne 0) {
return $false
}
if ($status -eq "healthy" -or $status -eq "none") {
return $true
}
Start-Sleep -Seconds 5
}
return $false
}
Invoke-Compose -Args @("down", "-v", "--remove-orphans") | Out-Null
$upExitCode = Invoke-Compose -Args @("up", "-d", "--wait", "--wait-timeout", $WaitTimeoutSeconds.ToString()) -IgnoreExitCode
if ($upExitCode -ne 0) {
Write-Warning "docker compose up returned exit code $upExitCode. Running unhealthy-service recovery."
}
for ($attempt = 1; $attempt -le $RecoveryAttempts; $attempt++) {
$unhealthyContainers = @(Get-UnhealthyContainers)
if ($unhealthyContainers.Count -eq 0) {
break
}
Write-Warning "Recovery attempt ${attempt}: unhealthy containers detected: $($unhealthyContainers -join ', ')"
$services = New-Object System.Collections.Generic.HashSet[string]([System.StringComparer]::OrdinalIgnoreCase)
foreach ($containerName in $unhealthyContainers) {
$serviceName = Get-ComposeServiceName -ContainerName $containerName
if (-not [string]::IsNullOrWhiteSpace($serviceName)) {
[void]$services.Add($serviceName)
}
}
foreach ($serviceName in $services) {
Write-Host "Recreating service: $serviceName"
Invoke-Compose -Args @("up", "-d", "--force-recreate", "--no-deps", $serviceName) | Out-Null
}
foreach ($containerName in $unhealthyContainers) {
[void](Wait-ForContainerHealth -ContainerName $containerName -TimeoutSeconds $RecoveryWaitSeconds)
}
}
$remainingUnhealthy = @(Get-UnhealthyContainers)
if ($remainingUnhealthy.Count -gt 0) {
throw "Redeploy completed with unresolved unhealthy containers: $($remainingUnhealthy -join ', ')"
}
if (-not $SkipHeaderSearchSmoke) {
$headerSearchSmokeScript = Join-Path -Path $PSScriptRoot -ChildPath "header-search-smoke.ps1"
if (-not (Test-Path -LiteralPath $headerSearchSmokeScript)) {
throw "Header search smoke script not found: $headerSearchSmokeScript"
}
Write-Host "Running header search route smoke checks..."
& $headerSearchSmokeScript
}
Write-Host "Redeploy complete for mode '$Mode'."