feat: Add VEX compact fixture and implement offline verifier for Findings Ledger exports
- Introduced a new VEX compact fixture for testing purposes. - Implemented `verify_export.py` script to validate Findings Ledger exports, ensuring deterministic ordering and applying redaction manifests. - Added a lightweight stub `HarnessRunner` for unit tests to validate ledger hashing expectations. - Documented tasks related to the Mirror Creator. - Created models for entropy signals and implemented the `EntropyPenaltyCalculator` to compute penalties based on scanner outputs. - Developed unit tests for `EntropyPenaltyCalculator` to ensure correct penalty calculations and handling of edge cases. - Added tests for symbol ID normalization in the reachability scanner. - Enhanced console status service with comprehensive unit tests for connection handling and error recovery. - Included Cosign tool version 2.6.0 with checksums for various platforms.
This commit is contained in:
@@ -5,18 +5,21 @@ Purpose: measure basic graph load/adjacency build and shallow path exploration o
|
||||
## Fixtures
|
||||
- Use interim synthetic fixtures under `samples/graph/interim/graph-50k` or `graph-100k`.
|
||||
- Each fixture includes `nodes.ndjson`, `edges.ndjson`, and `manifest.json` with hashes/counts.
|
||||
- Optional overlay: drop `overlay.ndjson` next to the fixture (or set `overlay.path` in `manifest.json`) to apply extra edges/layers; hashes are captured in results.
|
||||
|
||||
## Usage
|
||||
```bash
|
||||
python graph_bench.py \
|
||||
--fixture ../../../samples/graph/interim/graph-50k \
|
||||
--output results/graph-50k.json \
|
||||
--samples 100
|
||||
--samples 100 \
|
||||
--overlay ../../../samples/graph/interim/graph-50k/overlay.ndjson # optional
|
||||
```
|
||||
|
||||
Outputs a JSON summary with:
|
||||
- `nodes`, `edges`
|
||||
- `build_ms` — time to build adjacency (ms)
|
||||
- `overlay_ms` — time to apply overlay (0 when absent), plus counts and SHA under `overlay.*`
|
||||
- `bfs_ms` — total time for 3-depth BFS over sampled nodes
|
||||
- `avg_reach_3`, `max_reach_3` — nodes reached within depth 3
|
||||
- `manifest` — copied from fixture for traceability
|
||||
|
||||
Binary file not shown.
@@ -9,10 +9,11 @@ no network, and fixed seeds for reproducibility.
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
def load_ndjson(path: Path):
|
||||
@@ -42,6 +43,52 @@ def build_graph(nodes_path: Path, edges_path: Path) -> Tuple[Dict[str, List[str]
|
||||
return adjacency, edge_count
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def apply_overlay(adjacency: Dict[str, List[str]], overlay_path: Path) -> Tuple[int, int]:
|
||||
"""
|
||||
Apply overlay edges to the adjacency map.
|
||||
|
||||
Overlay file format (NDJSON): {"source": "nodeA", "target": "nodeB"}
|
||||
Unknown keys are ignored. New nodes are added with empty adjacency to keep
|
||||
BFS deterministic. Duplicate edges are de-duplicated.
|
||||
"""
|
||||
|
||||
if not overlay_path.exists():
|
||||
return 0, 0
|
||||
|
||||
added_edges = 0
|
||||
introduced_nodes = set()
|
||||
for record in load_ndjson(overlay_path):
|
||||
source = record.get("source") or record.get("from")
|
||||
target = record.get("target") or record.get("to")
|
||||
if not source or not target:
|
||||
continue
|
||||
|
||||
if source not in adjacency:
|
||||
adjacency[source] = []
|
||||
introduced_nodes.add(source)
|
||||
if target not in adjacency:
|
||||
adjacency[target] = []
|
||||
introduced_nodes.add(target)
|
||||
|
||||
if target not in adjacency[source]:
|
||||
adjacency[source].append(target)
|
||||
added_edges += 1
|
||||
|
||||
# keep neighbor ordering deterministic
|
||||
for v in adjacency.values():
|
||||
v.sort()
|
||||
|
||||
return added_edges, len(introduced_nodes)
|
||||
|
||||
|
||||
def bfs_limited(adjacency: Dict[str, List[str]], start: str, max_depth: int = 3) -> int:
|
||||
visited = {start}
|
||||
frontier = [start]
|
||||
@@ -58,15 +105,41 @@ def bfs_limited(adjacency: Dict[str, List[str]], start: str, max_depth: int = 3)
|
||||
return len(visited)
|
||||
|
||||
|
||||
def run_bench(fixture_dir: Path, sample_size: int = 100) -> dict:
|
||||
def resolve_overlay_path(fixture_dir: Path, manifest: dict, explicit: Optional[Path]) -> Optional[Path]:
|
||||
if explicit:
|
||||
return explicit.resolve()
|
||||
|
||||
overlay_manifest = manifest.get("overlay") if isinstance(manifest, dict) else None
|
||||
if isinstance(overlay_manifest, dict):
|
||||
path_value = overlay_manifest.get("path")
|
||||
if path_value:
|
||||
candidate = Path(path_value)
|
||||
return candidate if candidate.is_absolute() else (fixture_dir / candidate)
|
||||
|
||||
default = fixture_dir / "overlay.ndjson"
|
||||
return default if default.exists() else None
|
||||
|
||||
|
||||
def run_bench(fixture_dir: Path, sample_size: int = 100, overlay_path: Optional[Path] = None) -> dict:
|
||||
nodes_path = fixture_dir / "nodes.ndjson"
|
||||
edges_path = fixture_dir / "edges.ndjson"
|
||||
manifest_path = fixture_dir / "manifest.json"
|
||||
|
||||
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
|
||||
overlay_resolved = resolve_overlay_path(fixture_dir, manifest, overlay_path)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
adjacency, edge_count = build_graph(nodes_path, edges_path)
|
||||
overlay_added = 0
|
||||
overlay_nodes = 0
|
||||
overlay_hash = None
|
||||
overlay_ms = 0.0
|
||||
|
||||
if overlay_resolved:
|
||||
t_overlay = time.perf_counter()
|
||||
overlay_added, overlay_nodes = apply_overlay(adjacency, overlay_resolved)
|
||||
overlay_ms = (time.perf_counter() - t_overlay) * 1000
|
||||
overlay_hash = _sha256(overlay_resolved)
|
||||
build_ms = (time.perf_counter() - t0) * 1000
|
||||
|
||||
# deterministic sample: first N node ids sorted
|
||||
@@ -83,13 +156,21 @@ def run_bench(fixture_dir: Path, sample_size: int = 100) -> dict:
|
||||
return {
|
||||
"fixture": fixture_dir.name,
|
||||
"nodes": len(adjacency),
|
||||
"edges": edge_count,
|
||||
"edges": edge_count + overlay_added,
|
||||
"build_ms": round(build_ms, 2),
|
||||
"overlay_ms": round(overlay_ms, 2),
|
||||
"bfs_ms": round(bfs_ms, 2),
|
||||
"bfs_samples": len(node_ids),
|
||||
"avg_reach_3": round(avg_reach, 2),
|
||||
"max_reach_3": max_reach,
|
||||
"manifest": manifest,
|
||||
"overlay": {
|
||||
"applied": overlay_resolved is not None,
|
||||
"added_edges": overlay_added,
|
||||
"introduced_nodes": overlay_nodes,
|
||||
"path": str(overlay_resolved) if overlay_resolved else None,
|
||||
"sha256": overlay_hash,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -98,13 +179,15 @@ def main() -> int:
|
||||
parser.add_argument("--fixture", required=True, help="Path to fixture directory (nodes.ndjson, edges.ndjson)")
|
||||
parser.add_argument("--output", required=True, help="Path to write results JSON")
|
||||
parser.add_argument("--samples", type=int, default=100, help="Number of starting nodes to sample deterministically")
|
||||
parser.add_argument("--overlay", help="Optional overlay NDJSON path; defaults to overlay.ndjson next to fixture or manifest overlay.path")
|
||||
args = parser.parse_args()
|
||||
|
||||
fixture_dir = Path(args.fixture).resolve()
|
||||
out_path = Path(args.output).resolve()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
result = run_bench(fixture_dir, sample_size=args.samples)
|
||||
explicit_overlay = Path(args.overlay).resolve() if args.overlay else None
|
||||
result = run_bench(fixture_dir, sample_size=args.samples, overlay_path=explicit_overlay)
|
||||
out_path.write_text(json.dumps(result, indent=2, sort_keys=True))
|
||||
print(f"Wrote results to {out_path}")
|
||||
return 0
|
||||
|
||||
@@ -6,6 +6,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${ROOT}/../../../.." && pwd)"
|
||||
FIXTURES_ROOT="${FIXTURES_ROOT:-${REPO_ROOT}/samples/graph/interim}"
|
||||
OUT_DIR="${OUT_DIR:-$ROOT/results}"
|
||||
OVERLAY_ROOT="${OVERLAY_ROOT:-${FIXTURES_ROOT}}"
|
||||
SAMPLES="${SAMPLES:-100}"
|
||||
|
||||
mkdir -p "${OUT_DIR}"
|
||||
@@ -15,7 +16,14 @@ run_one() {
|
||||
local name
|
||||
name="$(basename "${fixture}")"
|
||||
local out_file="${OUT_DIR}/${name}.json"
|
||||
python "${ROOT}/graph_bench.py" --fixture "${fixture}" --output "${out_file}" --samples "${SAMPLES}"
|
||||
local overlay_candidate="${OVERLAY_ROOT}/${name}/overlay.ndjson"
|
||||
|
||||
args=("--fixture" "${fixture}" "--output" "${out_file}" "--samples" "${SAMPLES}")
|
||||
if [[ -f "${overlay_candidate}" ]]; then
|
||||
args+=("--overlay" "${overlay_candidate}")
|
||||
fi
|
||||
|
||||
python "${ROOT}/graph_bench.py" "${args[@]}"
|
||||
}
|
||||
|
||||
run_one "${FIXTURES_ROOT}/graph-50k"
|
||||
|
||||
Binary file not shown.
63
src/Bench/StellaOps.Bench/Graph/tests/test_graph_bench.py
Normal file
63
src/Bench/StellaOps.Bench/Graph/tests/test_graph_bench.py
Normal file
@@ -0,0 +1,63 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
|
||||
class GraphBenchTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.tmp.name)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _write_ndjson(self, path: Path, records):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
for record in records:
|
||||
f.write(json.dumps(record))
|
||||
f.write("\n")
|
||||
|
||||
def test_overlay_edges_are_applied_and_counted(self):
|
||||
from graph_bench import run_bench
|
||||
|
||||
fixture = self.root / "fixture"
|
||||
fixture.mkdir()
|
||||
|
||||
self._write_ndjson(fixture / "nodes.ndjson", [{"id": "a"}, {"id": "b"}])
|
||||
self._write_ndjson(fixture / "edges.ndjson", [{"source": "a", "target": "b"}])
|
||||
self._write_ndjson(fixture / "overlay.ndjson", [{"source": "b", "target": "a"}])
|
||||
|
||||
result = run_bench(fixture, sample_size=2)
|
||||
|
||||
self.assertEqual(result["nodes"], 2)
|
||||
self.assertEqual(result["edges"], 2) # overlay added one edge
|
||||
self.assertTrue(result["overlay"]["applied"])
|
||||
self.assertEqual(result["overlay"]["added_edges"], 1)
|
||||
self.assertEqual(result["overlay"]["introduced_nodes"], 0)
|
||||
|
||||
def test_overlay_is_optional(self):
|
||||
from graph_bench import run_bench
|
||||
|
||||
fixture = self.root / "fixture-no-overlay"
|
||||
fixture.mkdir()
|
||||
|
||||
self._write_ndjson(fixture / "nodes.ndjson", [{"id": "x"}, {"id": "y"}])
|
||||
self._write_ndjson(fixture / "edges.ndjson", [{"source": "x", "target": "y"}])
|
||||
|
||||
result = run_bench(fixture, sample_size=2)
|
||||
|
||||
self.assertEqual(result["edges"], 1)
|
||||
self.assertFalse(result["overlay"]["applied"])
|
||||
self.assertEqual(result["overlay"]["added_edges"], 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -7,18 +7,57 @@
|
||||
*/
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import crypto from "crypto";
|
||||
|
||||
function readJson(p) {
|
||||
return JSON.parse(fs.readFileSync(p, "utf-8"));
|
||||
}
|
||||
|
||||
function buildPlan(scenarios, manifest, fixtureName) {
|
||||
function sha256File(filePath) {
|
||||
const hash = crypto.createHash("sha256");
|
||||
hash.update(fs.readFileSync(filePath));
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
function resolveOverlay(fixtureDir, manifest) {
|
||||
const manifestOverlay = manifest?.overlay?.path;
|
||||
const candidate = manifestOverlay
|
||||
? path.isAbsolute(manifestOverlay)
|
||||
? manifestOverlay
|
||||
: path.join(fixtureDir, manifestOverlay)
|
||||
: path.join(fixtureDir, "overlay.ndjson");
|
||||
|
||||
if (!fs.existsSync(candidate)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
path: candidate,
|
||||
sha256: sha256File(candidate),
|
||||
};
|
||||
}
|
||||
|
||||
function buildPlan(scenarios, manifest, fixtureName, fixtureDir) {
|
||||
const now = new Date().toISOString();
|
||||
const seed = process.env.UI_BENCH_SEED || "424242";
|
||||
const traceId =
|
||||
process.env.UI_BENCH_TRACE_ID ||
|
||||
(crypto.randomUUID ? crypto.randomUUID() : `trace-${Date.now()}`);
|
||||
const overlay = resolveOverlay(fixtureDir, manifest);
|
||||
|
||||
return {
|
||||
version: "1.0.0",
|
||||
fixture: fixtureName,
|
||||
manifestHash: manifest?.hashes || {},
|
||||
overlay,
|
||||
timestamp: now,
|
||||
seed,
|
||||
traceId,
|
||||
viewport: {
|
||||
width: 1280,
|
||||
height: 720,
|
||||
deviceScaleFactor: 1,
|
||||
},
|
||||
steps: scenarios.map((s, idx) => ({
|
||||
order: idx + 1,
|
||||
id: s.id,
|
||||
@@ -41,7 +80,12 @@ function main() {
|
||||
const manifest = fs.existsSync(manifestPath) ? readJson(manifestPath) : {};
|
||||
const scenarios = readJson(scenariosPath).scenarios || [];
|
||||
|
||||
const plan = buildPlan(scenarios, manifest, path.basename(fixtureDir));
|
||||
const plan = buildPlan(
|
||||
scenarios,
|
||||
manifest,
|
||||
path.basename(fixtureDir),
|
||||
fixtureDir
|
||||
);
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(outputPath, JSON.stringify(plan, null, 2));
|
||||
console.log(`Wrote plan to ${outputPath}`);
|
||||
|
||||
42
src/Bench/StellaOps.Bench/Graph/ui_bench_driver.test.mjs
Normal file
42
src/Bench/StellaOps.Bench/Graph/ui_bench_driver.test.mjs
Normal file
@@ -0,0 +1,42 @@
|
||||
import assert from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
test("ui bench driver emits overlay + seed metadata", () => {
|
||||
const tmp = fs.mkdtempSync(path.join(process.cwd(), "tmp-ui-bench-"));
|
||||
const fixtureDir = path.join(tmp, "fixture");
|
||||
fs.mkdirSync(fixtureDir, { recursive: true });
|
||||
|
||||
// minimal fixture files
|
||||
fs.writeFileSync(path.join(fixtureDir, "manifest.json"), JSON.stringify({ hashes: { nodes: "abc" } }));
|
||||
fs.writeFileSync(path.join(fixtureDir, "overlay.ndjson"), "{\"source\":\"a\",\"target\":\"b\"}\n");
|
||||
|
||||
const scenariosPath = path.join(tmp, "scenarios.json");
|
||||
fs.writeFileSync(
|
||||
scenariosPath,
|
||||
JSON.stringify({ version: "1.0.0", scenarios: [{ id: "load", name: "Load", steps: ["navigate"] }] })
|
||||
);
|
||||
|
||||
const outputPath = path.join(tmp, "plan.json");
|
||||
const env = { ...process.env, UI_BENCH_SEED: "1337", UI_BENCH_TRACE_ID: "trace-test" };
|
||||
const driverPath = path.join(__dirname, "ui_bench_driver.mjs");
|
||||
const result = spawnSync(process.execPath, [driverPath, fixtureDir, scenariosPath, outputPath], { env });
|
||||
assert.strictEqual(result.status, 0, result.stderr?.toString());
|
||||
|
||||
const plan = JSON.parse(fs.readFileSync(outputPath, "utf-8"));
|
||||
assert.strictEqual(plan.fixture, "fixture");
|
||||
assert.strictEqual(plan.seed, "1337");
|
||||
assert.strictEqual(plan.traceId, "trace-test");
|
||||
assert.ok(plan.overlay);
|
||||
assert.ok(plan.overlay.path.endsWith("overlay.ndjson"));
|
||||
assert.ok(plan.overlay.sha256);
|
||||
assert.deepStrictEqual(plan.viewport, { width: 1280, height: 720, deviceScaleFactor: 1 });
|
||||
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
@@ -4,6 +4,7 @@ Purpose: provide a deterministic, headless flow for measuring graph UI interacti
|
||||
|
||||
## Scope
|
||||
- Use synthetic fixtures under `samples/graph/interim/` until canonical SAMPLES-GRAPH-24-003 lands.
|
||||
- Optional overlay layer (`overlay.ndjson`) is loaded when present and toggled during the run to capture render/merge overhead.
|
||||
- Drive a deterministic sequence of interactions:
|
||||
1) Load graph canvas with specified fixture.
|
||||
2) Pan to node `pkg-000001`.
|
||||
@@ -11,7 +12,7 @@ Purpose: provide a deterministic, headless flow for measuring graph UI interacti
|
||||
4) Apply filter `name contains "package-0001"`.
|
||||
5) Select node, expand neighbors (depth 1), collapse.
|
||||
6) Toggle overlay layer (once available).
|
||||
- Capture timings: initial render, filter apply, expand/collapse, overlay toggle.
|
||||
- Capture timings: initial render, filter apply, expand/collapse, overlay toggle (when available).
|
||||
|
||||
## Determinism rules
|
||||
- Fixed seed for any randomized layouts (seed `424242`).
|
||||
|
||||
Reference in New Issue
Block a user