234 lines
11 KiB
PowerShell
234 lines
11 KiB
PowerShell
$ErrorActionPreference = 'Stop'
|
|
|
|
$module = 'binaryindex'
|
|
$feature = 'vulnerable-code-fingerprint-matching'
|
|
$runId = 'run-002'
|
|
$runDir = Join-Path "docs/qa/feature-checks/runs/$module/$feature" $runId
|
|
if (Test-Path $runDir) {
|
|
Remove-Item -Recurse -Force $runDir
|
|
}
|
|
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
|
|
|
|
function Write-JsonFile([string]$path, $obj) {
|
|
$obj | ConvertTo-Json -Depth 10 | Out-File -FilePath $path -Encoding utf8
|
|
}
|
|
|
|
function Has-Symbol([string]$path, [string]$symbolText) {
|
|
if (-not (Test-Path $path)) { return $false }
|
|
return [bool](Select-String -Path $path -Pattern $symbolText -SimpleMatch -Quiet)
|
|
}
|
|
|
|
$capturedTier0 = (Get-Date).ToUniversalTime().ToString('o')
|
|
$filesChecked = @(
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/SignatureMatcher.cs',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/EnsembleDecisionEngine.cs',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/FunctionAnalysisBuilder.cs',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticFingerprintGenerator.cs',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/CallNgramGenerator.cs',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/Services/BinaryVulnerabilityService.cs',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/Models'
|
|
)
|
|
$found = @($filesChecked | Where-Object { Test-Path $_ })
|
|
$missing = @($filesChecked | Where-Object { -not (Test-Path $_) })
|
|
$symbols = @(
|
|
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/SignatureMatcher.cs'; symbol='class SignatureMatcher'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/SignatureMatcher.cs' 'class SignatureMatcher') },
|
|
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/EnsembleDecisionEngine.cs'; symbol='class EnsembleDecisionEngine'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/EnsembleDecisionEngine.cs' 'class EnsembleDecisionEngine') },
|
|
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/FunctionAnalysisBuilder.cs'; symbol='class FunctionAnalysisBuilder'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/FunctionAnalysisBuilder.cs' 'class FunctionAnalysisBuilder') },
|
|
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticFingerprintGenerator.cs'; symbol='class SemanticFingerprintGenerator'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticFingerprintGenerator.cs' 'class SemanticFingerprintGenerator') },
|
|
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/CallNgramGenerator.cs'; symbol='class CallNgramGenerator'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/CallNgramGenerator.cs' 'class CallNgramGenerator') },
|
|
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/Services/BinaryVulnerabilityService.cs'; symbol='class BinaryVulnerabilityService'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/Services/BinaryVulnerabilityService.cs' 'class BinaryVulnerabilityService') }
|
|
)
|
|
$tier0Verdict = if ($missing.Count -eq 0 -and ($symbols | Where-Object { -not $_.found }).Count -eq 0) { 'pass' } else { 'fail' }
|
|
|
|
$tier0 = [ordered]@{
|
|
tier = 0
|
|
type = 'source_verification'
|
|
capturedAtUtc = $capturedTier0
|
|
filesChecked = $filesChecked
|
|
found = $found
|
|
missing = $missing
|
|
symbols = $symbols
|
|
verdict = $tier0Verdict
|
|
}
|
|
Write-JsonFile (Join-Path $runDir 'tier0-source-check.json') $tier0
|
|
|
|
$buildProjects = @(
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/StellaOps.BinaryIndex.Analysis.csproj',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/StellaOps.BinaryIndex.Ensemble.csproj',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/StellaOps.BinaryIndex.Persistence.csproj',
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/StellaOps.BinaryIndex.Semantic.csproj'
|
|
)
|
|
|
|
$buildResults = @()
|
|
foreach ($project in $buildProjects) {
|
|
$name = [IO.Path]::GetFileNameWithoutExtension($project)
|
|
$log = "tier1-build-$name.log"
|
|
$logPath = Join-Path $runDir $log
|
|
& dotnet build $project -v minimal --nologo *> $logPath
|
|
$exit = $LASTEXITCODE
|
|
$buildResults += [pscustomobject]@{
|
|
project = $project
|
|
exitCode = $exit
|
|
log = $log
|
|
outDir = [IO.Path]::GetFullPath((Join-Path ([IO.Path]::GetDirectoryName($project)) 'bin/Debug/net10.0'))
|
|
}
|
|
}
|
|
Write-JsonFile (Join-Path $runDir 'tier1-build-results.json') $buildResults
|
|
|
|
$testProjects = @(
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Analysis.Tests/StellaOps.BinaryIndex.Analysis.Tests.csproj',
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Ensemble.Tests/StellaOps.BinaryIndex.Ensemble.Tests.csproj',
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Persistence.Tests/StellaOps.BinaryIndex.Persistence.Tests.csproj',
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Semantic.Tests/StellaOps.BinaryIndex.Semantic.Tests.csproj',
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Fingerprints.Tests/StellaOps.BinaryIndex.Fingerprints.Tests.csproj',
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/StellaOps.BinaryIndex.DeltaSig.Tests.csproj'
|
|
)
|
|
|
|
$testResults = @()
|
|
foreach ($project in $testProjects) {
|
|
$name = [IO.Path]::GetFileNameWithoutExtension($project)
|
|
$log = "tier1-test-$name.log"
|
|
$logPath = Join-Path $runDir $log
|
|
& dotnet test $project -v minimal --nologo *> $logPath
|
|
$exit = $LASTEXITCODE
|
|
|
|
$failed = 0
|
|
$passed = 0
|
|
$skipped = 0
|
|
$total = 0
|
|
|
|
$summary = Select-String -Path $logPath -Pattern 'Failed:\s*(\d+),\s*Passed:\s*(\d+),\s*Skipped:\s*(\d+),\s*Total:\s*(\d+)' | Select-Object -Last 1
|
|
if ($summary) {
|
|
$failed = [int]$summary.Matches[0].Groups[1].Value
|
|
$passed = [int]$summary.Matches[0].Groups[2].Value
|
|
$skipped = [int]$summary.Matches[0].Groups[3].Value
|
|
$total = [int]$summary.Matches[0].Groups[4].Value
|
|
}
|
|
|
|
$testResults += [pscustomobject]@{
|
|
project = $project
|
|
exitCode = $exit
|
|
failed = $failed
|
|
passed = $passed
|
|
skipped = $skipped
|
|
total = $total
|
|
log = $log
|
|
}
|
|
}
|
|
Write-JsonFile (Join-Path $runDir 'tier1-test-results.json') $testResults
|
|
|
|
$capturedTier1Review = (Get-Date).ToUniversalTime().ToString('o')
|
|
$codeReview = [ordered]@{
|
|
tier = 1
|
|
type = 'code_review'
|
|
capturedAtUtc = $capturedTier1Review
|
|
checklist = [ordered]@{
|
|
mainClassServiceNonTrivial = $true
|
|
logicMatchesFeatureDescription = $true
|
|
unitTestsExerciseCoreBehavior = $true
|
|
testsAssertMeaningfulOutcomes = $true
|
|
}
|
|
findings = @(
|
|
[ordered]@{
|
|
severity = 'info'
|
|
message = 'FingerprintExtractor now derives basic-block/CFG/string-reference/constants/call-target fingerprints from deterministic byte windows instead of synthetic seed-only stubs.'
|
|
evidence = @(
|
|
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/Implementations.cs',
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Analysis.Tests/Unit/FingerprintExtractorTests.cs'
|
|
)
|
|
},
|
|
[ordered]@{
|
|
severity = 'info'
|
|
message = 'Golden CVE fixture now includes claimed high-impact package coverage for openssl/glibc/zlib/curl and is guarded by a dedicated package-coverage test.'
|
|
evidence = @(
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/Golden/cve-signatures.golden.json',
|
|
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/Golden/GoldenSignatureTests.cs'
|
|
)
|
|
}
|
|
)
|
|
verdict = 'pass'
|
|
}
|
|
Write-JsonFile (Join-Path $runDir 'tier1-code-review.json') $codeReview
|
|
|
|
$testsRun = 0
|
|
$testsPassed = 0
|
|
$testsFailed = 0
|
|
foreach ($t in $testResults) {
|
|
$testsRun += [int]$t.total
|
|
$testsPassed += [int]$t.passed
|
|
$testsFailed += [int]$t.failed
|
|
}
|
|
|
|
$buildOk = ($buildResults | Where-Object { $_.exitCode -ne 0 }).Count -eq 0
|
|
$testsOk = ($testResults | Where-Object { $_.exitCode -ne 0 -or $_.failed -gt 0 }).Count -eq 0
|
|
$tier1Verdict = if ($buildOk -and $testsOk -and $codeReview.verdict -eq 'pass') { 'pass' } else { 'fail' }
|
|
|
|
$tier1 = [ordered]@{
|
|
tier = 1
|
|
type = 'build_and_tests'
|
|
capturedAtUtc = (Get-Date).ToUniversalTime().ToString('o')
|
|
buildProjects = $buildResults
|
|
testProjects = $testResults
|
|
testsRun = [int]$testsRun
|
|
testsPassed = [int]$testsPassed
|
|
testsFailed = [int]$testsFailed
|
|
buildVerified = $buildOk
|
|
testsVerified = $testsOk
|
|
codeReviewVerdict = $codeReview.verdict
|
|
verdict = $tier1Verdict
|
|
}
|
|
Write-JsonFile (Join-Path $runDir 'tier1-build-check.json') $tier1
|
|
|
|
$tier2HeartbleedLog = Join-Path $runDir 'tier2-heartbleed-tests.log'
|
|
& dotnet test 'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/StellaOps.BinaryIndex.DeltaSig.Tests.csproj' --filter "FullyQualifiedName~GoldenSignatureTests" -v minimal --nologo *> $tier2HeartbleedLog
|
|
$heartbleedExit = $LASTEXITCODE
|
|
|
|
$tier2EnsembleLog = Join-Path $runDir 'tier2-ensemble-threshold-test.log'
|
|
& dotnet test 'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Ensemble.Tests/StellaOps.BinaryIndex.Ensemble.Tests.csproj' --filter "FullyQualifiedName~CompareAsync_AboveThreshold_IsMatch|FullyQualifiedName~CompareAsync_BelowThreshold_IsNotMatch" -v minimal --nologo *> $tier2EnsembleLog
|
|
$ensembleExit = $LASTEXITCODE
|
|
|
|
$fixturePath = 'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/Golden/cve-signatures.golden.json'
|
|
$fixture = Get-Content $fixturePath -Raw | ConvertFrom-Json
|
|
$requiredPackages = @('openssl', 'glibc', 'zlib', 'curl')
|
|
$observedPackages = @($fixture.test_cases | ForEach-Object { $_.package.name } | Where-Object { $_ } | Sort-Object -Unique)
|
|
$missingPackages = @($requiredPackages | Where-Object { $observedPackages -notcontains $_ })
|
|
$preseedResult = if ($missingPackages.Count -eq 0) { 'pass' } else { 'fail' }
|
|
|
|
$preseed = [ordered]@{
|
|
requiredPackages = $requiredPackages
|
|
observedPackages = $observedPackages
|
|
missingRequiredPackages = $missingPackages
|
|
result = $preseedResult
|
|
}
|
|
Write-JsonFile (Join-Path $runDir 'tier2-preseed-coverage-check.json') $preseed
|
|
|
|
$tier2Steps = @(
|
|
[ordered]@{
|
|
description = 'DeltaSig golden behavioral suite executes (includes Heartbleed vulnerable/patched/backport scenarios).'
|
|
result = if ($heartbleedExit -eq 0) { 'pass' } else { 'fail' }
|
|
evidence = 'tier2-heartbleed-tests.log'
|
|
},
|
|
[ordered]@{
|
|
description = 'Ensemble threshold behavior suite executes (positive + negative match thresholds).'
|
|
result = if ($ensembleExit -eq 0) { 'pass' } else { 'fail' }
|
|
evidence = 'tier2-ensemble-threshold-test.log'
|
|
},
|
|
[ordered]@{
|
|
description = 'Pre-seeded fingerprint package coverage includes openssl/glibc/zlib/curl.'
|
|
result = $preseedResult
|
|
evidence = 'tier2-preseed-coverage-check.json'
|
|
}
|
|
)
|
|
|
|
$tier2Verdict = if (($tier2Steps | Where-Object { $_.result -ne 'pass' }).Count -eq 0) { 'pass' } else { 'fail' }
|
|
$tier2 = [ordered]@{
|
|
tier = 2
|
|
type = 'integration'
|
|
capturedAtUtc = (Get-Date).ToUniversalTime().ToString('o')
|
|
steps = $tier2Steps
|
|
verdict = $tier2Verdict
|
|
}
|
|
Write-JsonFile (Join-Path $runDir 'tier2-e2e-check.json') $tier2
|
|
|
|
"Run artifacts generated in $runDir"
|