feat: Add VEX compact fixture and implement offline verifier for Findings Ledger exports

- Introduced a new VEX compact fixture for testing purposes.
- Implemented `verify_export.py` script to validate Findings Ledger exports, ensuring deterministic ordering and applying redaction manifests.
- Added a lightweight stub `HarnessRunner` for unit tests to validate ledger hashing expectations.
- Documented tasks related to the Mirror Creator.
- Created models for entropy signals and implemented the `EntropyPenaltyCalculator` to compute penalties based on scanner outputs.
- Developed unit tests for `EntropyPenaltyCalculator` to ensure correct penalty calculations and handling of edge cases.
- Added tests for symbol ID normalization in the reachability scanner.
- Enhanced console status service with comprehensive unit tests for connection handling and error recovery.
- Included Cosign tool version 2.6.0 with checksums for various platforms.
This commit is contained in:
StellaOps Bot
2025-12-02 21:08:01 +02:00
parent 6d049905c7
commit 47168fec38
146 changed files with 4329 additions and 549 deletions

View File

@@ -5,18 +5,21 @@ Purpose: measure basic graph load/adjacency build and shallow path exploration o
## Fixtures
- Use interim synthetic fixtures under `samples/graph/interim/graph-50k` or `graph-100k`.
- Each fixture includes `nodes.ndjson`, `edges.ndjson`, and `manifest.json` with hashes/counts.
- Optional overlay: drop `overlay.ndjson` next to the fixture (or set `overlay.path` in `manifest.json`) to apply extra edges/layers; hashes are captured in results.
## Usage
```bash
python graph_bench.py \
--fixture ../../../samples/graph/interim/graph-50k \
--output results/graph-50k.json \
--samples 100
--samples 100 \
--overlay ../../../samples/graph/interim/graph-50k/overlay.ndjson # optional
```
Outputs a JSON summary with:
- `nodes`, `edges`
- `build_ms` — time to build adjacency (ms)
- `overlay_ms` — time to apply overlay (0 when absent), plus counts and SHA under `overlay.*`
- `bfs_ms` — total time for 3-depth BFS over sampled nodes
- `avg_reach_3`, `max_reach_3` — nodes reached within depth 3
- `manifest` — copied from fixture for traceability

View File

@@ -9,10 +9,11 @@ no network, and fixed seeds for reproducibility.
from __future__ import annotations
import argparse
import hashlib
import json
import time
from pathlib import Path
from typing import Dict, List, Tuple
from typing import Dict, List, Optional, Tuple
def load_ndjson(path: Path):
@@ -42,6 +43,52 @@ def build_graph(nodes_path: Path, edges_path: Path) -> Tuple[Dict[str, List[str]
return adjacency, edge_count
def _sha256(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(8192), b""):
h.update(chunk)
return h.hexdigest()
def apply_overlay(adjacency: Dict[str, List[str]], overlay_path: Path) -> Tuple[int, int]:
"""
Apply overlay edges to the adjacency map.
Overlay file format (NDJSON): {"source": "nodeA", "target": "nodeB"}
Unknown keys are ignored. New nodes are added with empty adjacency to keep
BFS deterministic. Duplicate edges are de-duplicated.
"""
if not overlay_path.exists():
return 0, 0
added_edges = 0
introduced_nodes = set()
for record in load_ndjson(overlay_path):
source = record.get("source") or record.get("from")
target = record.get("target") or record.get("to")
if not source or not target:
continue
if source not in adjacency:
adjacency[source] = []
introduced_nodes.add(source)
if target not in adjacency:
adjacency[target] = []
introduced_nodes.add(target)
if target not in adjacency[source]:
adjacency[source].append(target)
added_edges += 1
# keep neighbor ordering deterministic
for v in adjacency.values():
v.sort()
return added_edges, len(introduced_nodes)
def bfs_limited(adjacency: Dict[str, List[str]], start: str, max_depth: int = 3) -> int:
visited = {start}
frontier = [start]
@@ -58,15 +105,41 @@ def bfs_limited(adjacency: Dict[str, List[str]], start: str, max_depth: int = 3)
return len(visited)
def run_bench(fixture_dir: Path, sample_size: int = 100) -> dict:
def resolve_overlay_path(fixture_dir: Path, manifest: dict, explicit: Optional[Path]) -> Optional[Path]:
if explicit:
return explicit.resolve()
overlay_manifest = manifest.get("overlay") if isinstance(manifest, dict) else None
if isinstance(overlay_manifest, dict):
path_value = overlay_manifest.get("path")
if path_value:
candidate = Path(path_value)
return candidate if candidate.is_absolute() else (fixture_dir / candidate)
default = fixture_dir / "overlay.ndjson"
return default if default.exists() else None
def run_bench(fixture_dir: Path, sample_size: int = 100, overlay_path: Optional[Path] = None) -> dict:
nodes_path = fixture_dir / "nodes.ndjson"
edges_path = fixture_dir / "edges.ndjson"
manifest_path = fixture_dir / "manifest.json"
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
overlay_resolved = resolve_overlay_path(fixture_dir, manifest, overlay_path)
t0 = time.perf_counter()
adjacency, edge_count = build_graph(nodes_path, edges_path)
overlay_added = 0
overlay_nodes = 0
overlay_hash = None
overlay_ms = 0.0
if overlay_resolved:
t_overlay = time.perf_counter()
overlay_added, overlay_nodes = apply_overlay(adjacency, overlay_resolved)
overlay_ms = (time.perf_counter() - t_overlay) * 1000
overlay_hash = _sha256(overlay_resolved)
build_ms = (time.perf_counter() - t0) * 1000
# deterministic sample: first N node ids sorted
@@ -83,13 +156,21 @@ def run_bench(fixture_dir: Path, sample_size: int = 100) -> dict:
return {
"fixture": fixture_dir.name,
"nodes": len(adjacency),
"edges": edge_count,
"edges": edge_count + overlay_added,
"build_ms": round(build_ms, 2),
"overlay_ms": round(overlay_ms, 2),
"bfs_ms": round(bfs_ms, 2),
"bfs_samples": len(node_ids),
"avg_reach_3": round(avg_reach, 2),
"max_reach_3": max_reach,
"manifest": manifest,
"overlay": {
"applied": overlay_resolved is not None,
"added_edges": overlay_added,
"introduced_nodes": overlay_nodes,
"path": str(overlay_resolved) if overlay_resolved else None,
"sha256": overlay_hash,
},
}
@@ -98,13 +179,15 @@ def main() -> int:
parser.add_argument("--fixture", required=True, help="Path to fixture directory (nodes.ndjson, edges.ndjson)")
parser.add_argument("--output", required=True, help="Path to write results JSON")
parser.add_argument("--samples", type=int, default=100, help="Number of starting nodes to sample deterministically")
parser.add_argument("--overlay", help="Optional overlay NDJSON path; defaults to overlay.ndjson next to fixture or manifest overlay.path")
args = parser.parse_args()
fixture_dir = Path(args.fixture).resolve()
out_path = Path(args.output).resolve()
out_path.parent.mkdir(parents=True, exist_ok=True)
result = run_bench(fixture_dir, sample_size=args.samples)
explicit_overlay = Path(args.overlay).resolve() if args.overlay else None
result = run_bench(fixture_dir, sample_size=args.samples, overlay_path=explicit_overlay)
out_path.write_text(json.dumps(result, indent=2, sort_keys=True))
print(f"Wrote results to {out_path}")
return 0

View File

@@ -6,6 +6,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${ROOT}/../../../.." && pwd)"
FIXTURES_ROOT="${FIXTURES_ROOT:-${REPO_ROOT}/samples/graph/interim}"
OUT_DIR="${OUT_DIR:-$ROOT/results}"
OVERLAY_ROOT="${OVERLAY_ROOT:-${FIXTURES_ROOT}}"
SAMPLES="${SAMPLES:-100}"
mkdir -p "${OUT_DIR}"
@@ -15,7 +16,14 @@ run_one() {
local name
name="$(basename "${fixture}")"
local out_file="${OUT_DIR}/${name}.json"
python "${ROOT}/graph_bench.py" --fixture "${fixture}" --output "${out_file}" --samples "${SAMPLES}"
local overlay_candidate="${OVERLAY_ROOT}/${name}/overlay.ndjson"
args=("--fixture" "${fixture}" "--output" "${out_file}" "--samples" "${SAMPLES}")
if [[ -f "${overlay_candidate}" ]]; then
args+=("--overlay" "${overlay_candidate}")
fi
python "${ROOT}/graph_bench.py" "${args[@]}"
}
run_one "${FIXTURES_ROOT}/graph-50k"

View File

@@ -0,0 +1,63 @@
import json
import sys
import tempfile
from pathlib import Path
import unittest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
class GraphBenchTests(unittest.TestCase):
def setUp(self) -> None:
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
def tearDown(self) -> None:
self.tmp.cleanup()
def _write_ndjson(self, path: Path, records):
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as f:
for record in records:
f.write(json.dumps(record))
f.write("\n")
def test_overlay_edges_are_applied_and_counted(self):
from graph_bench import run_bench
fixture = self.root / "fixture"
fixture.mkdir()
self._write_ndjson(fixture / "nodes.ndjson", [{"id": "a"}, {"id": "b"}])
self._write_ndjson(fixture / "edges.ndjson", [{"source": "a", "target": "b"}])
self._write_ndjson(fixture / "overlay.ndjson", [{"source": "b", "target": "a"}])
result = run_bench(fixture, sample_size=2)
self.assertEqual(result["nodes"], 2)
self.assertEqual(result["edges"], 2) # overlay added one edge
self.assertTrue(result["overlay"]["applied"])
self.assertEqual(result["overlay"]["added_edges"], 1)
self.assertEqual(result["overlay"]["introduced_nodes"], 0)
def test_overlay_is_optional(self):
from graph_bench import run_bench
fixture = self.root / "fixture-no-overlay"
fixture.mkdir()
self._write_ndjson(fixture / "nodes.ndjson", [{"id": "x"}, {"id": "y"}])
self._write_ndjson(fixture / "edges.ndjson", [{"source": "x", "target": "y"}])
result = run_bench(fixture, sample_size=2)
self.assertEqual(result["edges"], 1)
self.assertFalse(result["overlay"]["applied"])
self.assertEqual(result["overlay"]["added_edges"], 0)
if __name__ == "__main__":
unittest.main()

View File

@@ -7,18 +7,57 @@
*/
import fs from "fs";
import path from "path";
import crypto from "crypto";
function readJson(p) {
return JSON.parse(fs.readFileSync(p, "utf-8"));
}
function buildPlan(scenarios, manifest, fixtureName) {
function sha256File(filePath) {
const hash = crypto.createHash("sha256");
hash.update(fs.readFileSync(filePath));
return hash.digest("hex");
}
function resolveOverlay(fixtureDir, manifest) {
const manifestOverlay = manifest?.overlay?.path;
const candidate = manifestOverlay
? path.isAbsolute(manifestOverlay)
? manifestOverlay
: path.join(fixtureDir, manifestOverlay)
: path.join(fixtureDir, "overlay.ndjson");
if (!fs.existsSync(candidate)) {
return null;
}
return {
path: candidate,
sha256: sha256File(candidate),
};
}
function buildPlan(scenarios, manifest, fixtureName, fixtureDir) {
const now = new Date().toISOString();
const seed = process.env.UI_BENCH_SEED || "424242";
const traceId =
process.env.UI_BENCH_TRACE_ID ||
(crypto.randomUUID ? crypto.randomUUID() : `trace-${Date.now()}`);
const overlay = resolveOverlay(fixtureDir, manifest);
return {
version: "1.0.0",
fixture: fixtureName,
manifestHash: manifest?.hashes || {},
overlay,
timestamp: now,
seed,
traceId,
viewport: {
width: 1280,
height: 720,
deviceScaleFactor: 1,
},
steps: scenarios.map((s, idx) => ({
order: idx + 1,
id: s.id,
@@ -41,7 +80,12 @@ function main() {
const manifest = fs.existsSync(manifestPath) ? readJson(manifestPath) : {};
const scenarios = readJson(scenariosPath).scenarios || [];
const plan = buildPlan(scenarios, manifest, path.basename(fixtureDir));
const plan = buildPlan(
scenarios,
manifest,
path.basename(fixtureDir),
fixtureDir
);
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
fs.writeFileSync(outputPath, JSON.stringify(plan, null, 2));
console.log(`Wrote plan to ${outputPath}`);

View File

@@ -0,0 +1,42 @@
import assert from "node:assert";
import { test } from "node:test";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawnSync } from "node:child_process";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
test("ui bench driver emits overlay + seed metadata", () => {
const tmp = fs.mkdtempSync(path.join(process.cwd(), "tmp-ui-bench-"));
const fixtureDir = path.join(tmp, "fixture");
fs.mkdirSync(fixtureDir, { recursive: true });
// minimal fixture files
fs.writeFileSync(path.join(fixtureDir, "manifest.json"), JSON.stringify({ hashes: { nodes: "abc" } }));
fs.writeFileSync(path.join(fixtureDir, "overlay.ndjson"), "{\"source\":\"a\",\"target\":\"b\"}\n");
const scenariosPath = path.join(tmp, "scenarios.json");
fs.writeFileSync(
scenariosPath,
JSON.stringify({ version: "1.0.0", scenarios: [{ id: "load", name: "Load", steps: ["navigate"] }] })
);
const outputPath = path.join(tmp, "plan.json");
const env = { ...process.env, UI_BENCH_SEED: "1337", UI_BENCH_TRACE_ID: "trace-test" };
const driverPath = path.join(__dirname, "ui_bench_driver.mjs");
const result = spawnSync(process.execPath, [driverPath, fixtureDir, scenariosPath, outputPath], { env });
assert.strictEqual(result.status, 0, result.stderr?.toString());
const plan = JSON.parse(fs.readFileSync(outputPath, "utf-8"));
assert.strictEqual(plan.fixture, "fixture");
assert.strictEqual(plan.seed, "1337");
assert.strictEqual(plan.traceId, "trace-test");
assert.ok(plan.overlay);
assert.ok(plan.overlay.path.endsWith("overlay.ndjson"));
assert.ok(plan.overlay.sha256);
assert.deepStrictEqual(plan.viewport, { width: 1280, height: 720, deviceScaleFactor: 1 });
fs.rmSync(tmp, { recursive: true, force: true });
});

View File

@@ -4,6 +4,7 @@ Purpose: provide a deterministic, headless flow for measuring graph UI interacti
## Scope
- Use synthetic fixtures under `samples/graph/interim/` until canonical SAMPLES-GRAPH-24-003 lands.
- Optional overlay layer (`overlay.ndjson`) is loaded when present and toggled during the run to capture render/merge overhead.
- Drive a deterministic sequence of interactions:
1) Load graph canvas with specified fixture.
2) Pan to node `pkg-000001`.
@@ -11,7 +12,7 @@ Purpose: provide a deterministic, headless flow for measuring graph UI interacti
4) Apply filter `name contains "package-0001"`.
5) Select node, expand neighbors (depth 1), collapse.
6) Toggle overlay layer (once available).
- Capture timings: initial render, filter apply, expand/collapse, overlay toggle.
- Capture timings: initial render, filter apply, expand/collapse, overlay toggle (when available).
## Determinism rules
- Fixed seed for any randomized layouts (seed `424242`).