81 lines
9.0 KiB
JSON
81 lines
9.0 KiB
JSON
{
|
|
"module": "bench",
|
|
"featureCount": 3,
|
|
"lastUpdatedUtc": "2026-02-11T10:52:19.3903646Z",
|
|
"features": {
|
|
"benchmark-harness": {
|
|
"status": "done",
|
|
"tier": 2,
|
|
"retryCount": 2,
|
|
"sourceVerified": true,
|
|
"buildVerified": true,
|
|
"e2eVerified": true,
|
|
"skipReason": null,
|
|
"lastRunId": "run-005",
|
|
"lastUpdatedUtc": "2026-02-11T10:52:19.3903646Z",
|
|
"featureFile": "docs/features/checked/bench/benchmark-harness.md",
|
|
"notes": [
|
|
"[2026-02-11T10:29:06.7424460Z] checking: Started run-002 Tier 0/1/2 verification for benchmark-harness with full CLI behavioral replay.",
|
|
"[2026-02-11T10:29:43.8347907Z] checking: Started fresh run-002 Tier 0/1/2 verification for benchmark-harness because run-001 lacked terminal Tier 2 artifact.",
|
|
"[2026-02-11T10:36:53.1419686Z] done: Tier 0/1/2 verification passed in run-002 with fresh benchmark CLI evidence; feature moved from unchecked to checked.",
|
|
"[2026-02-11T10:42:24.7769912Z] triaged: Tier 2 PolicyEngine command failed on default allocation cap; classified as config threshold mismatch for local verification host.",
|
|
"[2026-02-11T10:42:24.7769912Z] retesting: Replayed Tier 2 with run-local relaxed PolicyEngine scenario thresholds and regenerated fresh command evidence.",
|
|
"[2026-02-11T10:42:24.7769912Z] done: benchmark-harness passed Tier 0/1/2 and remains in checked with run-002 terminal evidence.",
|
|
"[2026-02-11T10:45:28.9703527+00:00] checking: Started run-003 Tier 0/1/2 verification to capture fresh complete CLI evidence.",
|
|
"[2026-02-11T10:45:28.9703527+00:00] failed: Tier 2 PolicyEngine benchmark failed due invalid policy config path and strict scenario-level allocation cap on host.",
|
|
"[2026-02-11T10:45:28.9703527+00:00] triaged: Classified as config/threshold mismatch; benchmark behavior itself was implemented.",
|
|
"[2026-02-11T10:45:28.9703527+00:00] confirmed: Root cause confirmed from command output and config inspection.",
|
|
"[2026-02-11T10:45:28.9703527+00:00] fixing: Added benchmark-default policy fixture and corrected PolicyEngine benchmark config policyPath.",
|
|
"[2026-02-11T10:45:28.9703527+00:00] retesting: Re-ran Tier 1 and Tier 2 with fresh run-003 artifacts including positive+negative CLI replay.",
|
|
"[2026-02-11T10:45:28.9703527+00:00] done: benchmark-harness verified with run-003 terminal evidence and remains in checked.",
|
|
"[2026-02-11T10:52:19.3903646Z] retesting: Executed fresh run-005 Tier 0/1/2 benchmark-harness replay with corrected PolicyEngine benchmark fixture and full CLI evidence.",
|
|
"[2026-02-11T10:52:19.3903646Z] done: benchmark-harness terminal verification set to run-005 after clean Tier 0/1/2 pass including negative-path CLI assertion."
|
|
]
|
|
},
|
|
"reachability-benchmarks-with-ground-truth-datasets": {
|
|
"status": "done",
|
|
"tier": 2,
|
|
"retryCount": 1,
|
|
"sourceVerified": true,
|
|
"buildVerified": true,
|
|
"e2eVerified": true,
|
|
"skipReason": null,
|
|
"lastRunId": "run-002",
|
|
"lastUpdatedUtc": "2026-02-11T10:11:23.4266418Z",
|
|
"featureFile": "docs/features/checked/bench/reachability-benchmarks-with-ground-truth-datasets.md",
|
|
"notes": [
|
|
"[2026-02-11T10:11:23.4266418Z] checking: Started Tier 0/1/2 verification for bench feature reachability-benchmarks-with-ground-truth-datasets.",
|
|
"[2026-02-11T10:11:23.4266418Z] failed: run-001 Tier 2 failed with unsupported analyzer exception from ScenarioRunnerFactory.CreateFactory.",
|
|
"[2026-02-11T10:11:23.4266418Z] triaged: Root cause identified in ScenarioRunners analyzer factory mapping.",
|
|
"[2026-02-11T10:11:23.4266418Z] confirmed: Failure cause confirmed and approved for fix.",
|
|
"[2026-02-11T10:11:23.4266418Z] fixing: Implemented analyzer mappings and added unit tests for supported/unsupported IDs.",
|
|
"[2026-02-11T10:11:23.4266418Z] retesting: run-002 executed with fresh Tier 0/1/2 artifacts.",
|
|
"[2026-02-11T10:11:23.4266418Z] done: Feature moved to checked after passing run-002 Tier 0/1/2 with fresh CLI evidence."
|
|
]
|
|
},
|
|
"vendor-comparison-scanner-parity-tracking": {
|
|
"status": "done",
|
|
"tier": 2,
|
|
"retryCount": 0,
|
|
"sourceVerified": true,
|
|
"buildVerified": true,
|
|
"e2eVerified": true,
|
|
"skipReason": null,
|
|
"lastRunId": "run-001",
|
|
"lastUpdatedUtc": "2026-02-11T10:40:08.7502595Z",
|
|
"featureFile": "docs/features/checked/bench/vendor-comparison-scanner-parity-tracking.md",
|
|
"notes": [
|
|
"[2026-02-11T10:36:53.1419686Z] checking: Started Tier 0/1/2 verification for vendor-comparison-scanner-parity-tracking.",
|
|
"[2026-02-11T10:40:08.7502595Z] done: Tier 0/1/2 verification passed in run-001 with scanner parity benchmark evidence; feature moved from unchecked to checked."
|
|
]
|
|
}
|
|
},
|
|
"summary": {
|
|
"passed": 3,
|
|
"failed": 0,
|
|
"blocked": 0,
|
|
"skipped": 0,
|
|
"done": 3
|
|
}
|
|
}
|