save checkpoint: save features
This commit is contained in:
master
233
tmp_binaryindex_vulncode_run002.ps1
Normal file
233
tmp_binaryindex_vulncode_run002.ps1
Normal file
@@ -0,0 +1,233 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$module = 'binaryindex'
|
||||
$feature = 'vulnerable-code-fingerprint-matching'
|
||||
$runId = 'run-002'
|
||||
$runDir = Join-Path "docs/qa/feature-checks/runs/$module/$feature" $runId
|
||||
if (Test-Path $runDir) {
|
||||
Remove-Item -Recurse -Force $runDir
|
||||
}
|
||||
New-Item -ItemType Directory -Force -Path $runDir | Out-Null
|
||||
|
||||
function Write-JsonFile([string]$path, $obj) {
|
||||
$obj | ConvertTo-Json -Depth 10 | Out-File -FilePath $path -Encoding utf8
|
||||
}
|
||||
|
||||
function Has-Symbol([string]$path, [string]$symbolText) {
|
||||
if (-not (Test-Path $path)) { return $false }
|
||||
return [bool](Select-String -Path $path -Pattern $symbolText -SimpleMatch -Quiet)
|
||||
}
|
||||
|
||||
$capturedTier0 = (Get-Date).ToUniversalTime().ToString('o')
|
||||
$filesChecked = @(
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/SignatureMatcher.cs',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/EnsembleDecisionEngine.cs',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/FunctionAnalysisBuilder.cs',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticFingerprintGenerator.cs',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/CallNgramGenerator.cs',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/Services/BinaryVulnerabilityService.cs',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/Models'
|
||||
)
|
||||
$found = @($filesChecked | Where-Object { Test-Path $_ })
|
||||
$missing = @($filesChecked | Where-Object { -not (Test-Path $_) })
|
||||
$symbols = @(
|
||||
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/SignatureMatcher.cs'; symbol='class SignatureMatcher'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/SignatureMatcher.cs' 'class SignatureMatcher') },
|
||||
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/EnsembleDecisionEngine.cs'; symbol='class EnsembleDecisionEngine'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/EnsembleDecisionEngine.cs' 'class EnsembleDecisionEngine') },
|
||||
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/FunctionAnalysisBuilder.cs'; symbol='class FunctionAnalysisBuilder'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/FunctionAnalysisBuilder.cs' 'class FunctionAnalysisBuilder') },
|
||||
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticFingerprintGenerator.cs'; symbol='class SemanticFingerprintGenerator'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticFingerprintGenerator.cs' 'class SemanticFingerprintGenerator') },
|
||||
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/CallNgramGenerator.cs'; symbol='class CallNgramGenerator'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/CallNgramGenerator.cs' 'class CallNgramGenerator') },
|
||||
[pscustomobject]@{ path='src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/Services/BinaryVulnerabilityService.cs'; symbol='class BinaryVulnerabilityService'; found=(Has-Symbol 'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/Services/BinaryVulnerabilityService.cs' 'class BinaryVulnerabilityService') }
|
||||
)
|
||||
$tier0Verdict = if ($missing.Count -eq 0 -and ($symbols | Where-Object { -not $_.found }).Count -eq 0) { 'pass' } else { 'fail' }
|
||||
|
||||
$tier0 = [ordered]@{
|
||||
tier = 0
|
||||
type = 'source_verification'
|
||||
capturedAtUtc = $capturedTier0
|
||||
filesChecked = $filesChecked
|
||||
found = $found
|
||||
missing = $missing
|
||||
symbols = $symbols
|
||||
verdict = $tier0Verdict
|
||||
}
|
||||
Write-JsonFile (Join-Path $runDir 'tier0-source-check.json') $tier0
|
||||
|
||||
$buildProjects = @(
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/StellaOps.BinaryIndex.Analysis.csproj',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/StellaOps.BinaryIndex.Ensemble.csproj',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Persistence/StellaOps.BinaryIndex.Persistence.csproj',
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/StellaOps.BinaryIndex.Semantic.csproj'
|
||||
)
|
||||
|
||||
$buildResults = @()
|
||||
foreach ($project in $buildProjects) {
|
||||
$name = [IO.Path]::GetFileNameWithoutExtension($project)
|
||||
$log = "tier1-build-$name.log"
|
||||
$logPath = Join-Path $runDir $log
|
||||
& dotnet build $project -v minimal --nologo *> $logPath
|
||||
$exit = $LASTEXITCODE
|
||||
$buildResults += [pscustomobject]@{
|
||||
project = $project
|
||||
exitCode = $exit
|
||||
log = $log
|
||||
outDir = [IO.Path]::GetFullPath((Join-Path ([IO.Path]::GetDirectoryName($project)) 'bin/Debug/net10.0'))
|
||||
}
|
||||
}
|
||||
Write-JsonFile (Join-Path $runDir 'tier1-build-results.json') $buildResults
|
||||
|
||||
$testProjects = @(
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Analysis.Tests/StellaOps.BinaryIndex.Analysis.Tests.csproj',
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Ensemble.Tests/StellaOps.BinaryIndex.Ensemble.Tests.csproj',
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Persistence.Tests/StellaOps.BinaryIndex.Persistence.Tests.csproj',
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Semantic.Tests/StellaOps.BinaryIndex.Semantic.Tests.csproj',
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Fingerprints.Tests/StellaOps.BinaryIndex.Fingerprints.Tests.csproj',
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/StellaOps.BinaryIndex.DeltaSig.Tests.csproj'
|
||||
)
|
||||
|
||||
$testResults = @()
|
||||
foreach ($project in $testProjects) {
|
||||
$name = [IO.Path]::GetFileNameWithoutExtension($project)
|
||||
$log = "tier1-test-$name.log"
|
||||
$logPath = Join-Path $runDir $log
|
||||
& dotnet test $project -v minimal --nologo *> $logPath
|
||||
$exit = $LASTEXITCODE
|
||||
|
||||
$failed = 0
|
||||
$passed = 0
|
||||
$skipped = 0
|
||||
$total = 0
|
||||
|
||||
$summary = Select-String -Path $logPath -Pattern 'Failed:\s*(\d+),\s*Passed:\s*(\d+),\s*Skipped:\s*(\d+),\s*Total:\s*(\d+)' | Select-Object -Last 1
|
||||
if ($summary) {
|
||||
$failed = [int]$summary.Matches[0].Groups[1].Value
|
||||
$passed = [int]$summary.Matches[0].Groups[2].Value
|
||||
$skipped = [int]$summary.Matches[0].Groups[3].Value
|
||||
$total = [int]$summary.Matches[0].Groups[4].Value
|
||||
}
|
||||
|
||||
$testResults += [pscustomobject]@{
|
||||
project = $project
|
||||
exitCode = $exit
|
||||
failed = $failed
|
||||
passed = $passed
|
||||
skipped = $skipped
|
||||
total = $total
|
||||
log = $log
|
||||
}
|
||||
}
|
||||
Write-JsonFile (Join-Path $runDir 'tier1-test-results.json') $testResults
|
||||
|
||||
$capturedTier1Review = (Get-Date).ToUniversalTime().ToString('o')
|
||||
$codeReview = [ordered]@{
|
||||
tier = 1
|
||||
type = 'code_review'
|
||||
capturedAtUtc = $capturedTier1Review
|
||||
checklist = [ordered]@{
|
||||
mainClassServiceNonTrivial = $true
|
||||
logicMatchesFeatureDescription = $true
|
||||
unitTestsExerciseCoreBehavior = $true
|
||||
testsAssertMeaningfulOutcomes = $true
|
||||
}
|
||||
findings = @(
|
||||
[ordered]@{
|
||||
severity = 'info'
|
||||
message = 'FingerprintExtractor now derives basic-block/CFG/string-reference/constants/call-target fingerprints from deterministic byte windows instead of synthetic seed-only stubs.'
|
||||
evidence = @(
|
||||
'src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Analysis/Implementations.cs',
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Analysis.Tests/Unit/FingerprintExtractorTests.cs'
|
||||
)
|
||||
},
|
||||
[ordered]@{
|
||||
severity = 'info'
|
||||
message = 'Golden CVE fixture now includes claimed high-impact package coverage for openssl/glibc/zlib/curl and is guarded by a dedicated package-coverage test.'
|
||||
evidence = @(
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/Golden/cve-signatures.golden.json',
|
||||
'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/Golden/GoldenSignatureTests.cs'
|
||||
)
|
||||
}
|
||||
)
|
||||
verdict = 'pass'
|
||||
}
|
||||
Write-JsonFile (Join-Path $runDir 'tier1-code-review.json') $codeReview
|
||||
|
||||
$testsRun = 0
|
||||
$testsPassed = 0
|
||||
$testsFailed = 0
|
||||
foreach ($t in $testResults) {
|
||||
$testsRun += [int]$t.total
|
||||
$testsPassed += [int]$t.passed
|
||||
$testsFailed += [int]$t.failed
|
||||
}
|
||||
|
||||
$buildOk = ($buildResults | Where-Object { $_.exitCode -ne 0 }).Count -eq 0
|
||||
$testsOk = ($testResults | Where-Object { $_.exitCode -ne 0 -or $_.failed -gt 0 }).Count -eq 0
|
||||
$tier1Verdict = if ($buildOk -and $testsOk -and $codeReview.verdict -eq 'pass') { 'pass' } else { 'fail' }
|
||||
|
||||
$tier1 = [ordered]@{
|
||||
tier = 1
|
||||
type = 'build_and_tests'
|
||||
capturedAtUtc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
buildProjects = $buildResults
|
||||
testProjects = $testResults
|
||||
testsRun = [int]$testsRun
|
||||
testsPassed = [int]$testsPassed
|
||||
testsFailed = [int]$testsFailed
|
||||
buildVerified = $buildOk
|
||||
testsVerified = $testsOk
|
||||
codeReviewVerdict = $codeReview.verdict
|
||||
verdict = $tier1Verdict
|
||||
}
|
||||
Write-JsonFile (Join-Path $runDir 'tier1-build-check.json') $tier1
|
||||
|
||||
$tier2HeartbleedLog = Join-Path $runDir 'tier2-heartbleed-tests.log'
|
||||
& dotnet test 'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/StellaOps.BinaryIndex.DeltaSig.Tests.csproj' --filter "FullyQualifiedName~GoldenSignatureTests" -v minimal --nologo *> $tier2HeartbleedLog
|
||||
$heartbleedExit = $LASTEXITCODE
|
||||
|
||||
$tier2EnsembleLog = Join-Path $runDir 'tier2-ensemble-threshold-test.log'
|
||||
& dotnet test 'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.Ensemble.Tests/StellaOps.BinaryIndex.Ensemble.Tests.csproj' --filter "FullyQualifiedName~CompareAsync_AboveThreshold_IsMatch|FullyQualifiedName~CompareAsync_BelowThreshold_IsNotMatch" -v minimal --nologo *> $tier2EnsembleLog
|
||||
$ensembleExit = $LASTEXITCODE
|
||||
|
||||
$fixturePath = 'src/BinaryIndex/__Tests/StellaOps.BinaryIndex.DeltaSig.Tests/Golden/cve-signatures.golden.json'
|
||||
$fixture = Get-Content $fixturePath -Raw | ConvertFrom-Json
|
||||
$requiredPackages = @('openssl', 'glibc', 'zlib', 'curl')
|
||||
$observedPackages = @($fixture.test_cases | ForEach-Object { $_.package.name } | Where-Object { $_ } | Sort-Object -Unique)
|
||||
$missingPackages = @($requiredPackages | Where-Object { $observedPackages -notcontains $_ })
|
||||
$preseedResult = if ($missingPackages.Count -eq 0) { 'pass' } else { 'fail' }
|
||||
|
||||
$preseed = [ordered]@{
|
||||
requiredPackages = $requiredPackages
|
||||
observedPackages = $observedPackages
|
||||
missingRequiredPackages = $missingPackages
|
||||
result = $preseedResult
|
||||
}
|
||||
Write-JsonFile (Join-Path $runDir 'tier2-preseed-coverage-check.json') $preseed
|
||||
|
||||
$tier2Steps = @(
|
||||
[ordered]@{
|
||||
description = 'DeltaSig golden behavioral suite executes (includes Heartbleed vulnerable/patched/backport scenarios).'
|
||||
result = if ($heartbleedExit -eq 0) { 'pass' } else { 'fail' }
|
||||
evidence = 'tier2-heartbleed-tests.log'
|
||||
},
|
||||
[ordered]@{
|
||||
description = 'Ensemble threshold behavior suite executes (positive + negative match thresholds).'
|
||||
result = if ($ensembleExit -eq 0) { 'pass' } else { 'fail' }
|
||||
evidence = 'tier2-ensemble-threshold-test.log'
|
||||
},
|
||||
[ordered]@{
|
||||
description = 'Pre-seeded fingerprint package coverage includes openssl/glibc/zlib/curl.'
|
||||
result = $preseedResult
|
||||
evidence = 'tier2-preseed-coverage-check.json'
|
||||
}
|
||||
)
|
||||
|
||||
$tier2Verdict = if (($tier2Steps | Where-Object { $_.result -ne 'pass' }).Count -eq 0) { 'pass' } else { 'fail' }
|
||||
$tier2 = [ordered]@{
|
||||
tier = 2
|
||||
type = 'integration'
|
||||
capturedAtUtc = (Get-Date).ToUniversalTime().ToString('o')
|
||||
steps = $tier2Steps
|
||||
verdict = $tier2Verdict
|
||||
}
|
||||
Write-JsonFile (Join-Path $runDir 'tier2-e2e-check.json') $tier2
|
||||
|
||||
"Run artifacts generated in $runDir"
|
||||
Reference in New Issue
Block a user