up

src/Bench/StellaOps.Bench/Signals/README.md

@@ -0,0 +1,24 @@
+# Reachability Bench
+
+Benchmarks the reachability scoring pipeline using offline synthetic fixtures.
+
+## Inputs
+- Callgraph fixtures: `docs/samples/signals/reachability/callgraph-10k.ndjson` and `callgraph-50k.ndjson`.
+- Runtime traces: `docs/samples/signals/reachability/runtime-10k.ndjson` and `runtime-50k.ndjson`.
+- Schema hash: `docs/benchmarks/signals/reachability-schema.json` (sha256 `aaa5c8ab5cc2fe91e50976fafd8c73597387ab9a881af6d5d9818d202beba24e`).
+
+## Running
+```bash
+python reachability_bench.py --callgraph ../../../../docs/samples/signals/reachability/callgraph-10k.ndjson --runtime ../../../../docs/samples/signals/reachability/runtime-10k.ndjson --output results/reachability-metrics-10k.ndjson --cache-output results/reachability-cache-10k.ndjson --threads 1 --seed 20250101
+```
+
+Swap the input paths for the 50k fixtures to exercise the larger dataset.
+
+## Output
+- Metrics NDJSON with fields: `run`, `startedAtUtc`, `functions`, `runtimeEvents`, `facts`, `durationMs`, `factsPerSec`, `p50MsPerNode`, `p95MsPerNode`, `p99MsPerNode`, `rssMb`, `managedMb`, `gcGen2`.
+- Cache NDJSON (`reachability-cache-*.ndjson`) with per-function reachability flags, fanout, and runtime counts for downstream policy benches.
+
+## Determinism
+- Processing order is sorted by runtime function id; graph traversal preserves deterministic queueing.
+- Single-threaded execution avoids nondeterministic scheduling.
+- Output JSON keys are sorted for stable diffs; timestamps use UTC ISO-8601.

src/Bench/StellaOps.Bench/Signals/__init__.py

@@ -0,0 +1 @@
+# Package marker for reachability bench.

src/Bench/StellaOps.Bench/Signals/reachability_bench.py

@@ -0,0 +1,151 @@
+"""Reachability scoring benchmark.
+
+Processes synthetic callgraph and runtime traces to measure facts/sec, latency,
+and memory. Outputs both metrics and a cache file consumable by policy benches.
+"""
+
+import argparse
+import gc
+import json
+import time
+from collections import deque
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Iterable, List, Tuple
+
+
+def percentile(values: List[float], pct: float) -> float:
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    k = (len(ordered) - 1) * (pct / 100.0)
+    lower = int(k)
+    upper = min(lower + 1, len(ordered) - 1)
+    if lower == upper:
+        return float(ordered[lower])
+    fraction = k - lower
+    return float(ordered[lower] + (ordered[upper] - ordered[lower]) * fraction)
+
+
+def load_ndjson(path: Path) -> List[dict]:
+    with path.open(encoding="utf-8") as handle:
+        return [json.loads(line) for line in handle if line.strip()]
+
+
+def build_graph(callgraph_records: Iterable[dict]) -> Dict[str, List[str]]:
+    graph: Dict[str, List[str]] = {}
+    for record in callgraph_records:
+        graph[record["function"]] = record.get("calls", [])
+    return graph
+
+
+def build_runtime(runtime_records: Iterable[dict]) -> Dict[str, int]:
+    runtime: Dict[str, int] = {}
+    for record in runtime_records:
+        runtime[record["function"]] = runtime.get(record["function"], 0) + int(record.get("count", 0))
+    return runtime
+
+
+def run_reachability(graph: Dict[str, List[str]], runtime: Dict[str, int]) -> Tuple[dict, List[dict]]:
+    import tracemalloc
+
+    started_at = datetime.now(timezone.utc).isoformat()
+    visited = set()
+    cache_records: List[dict] = []
+    timings_ms: List[float] = []
+    facts = 0
+
+    queue = deque(sorted(runtime.keys()))
+    gc.collect()
+    tracemalloc.start()
+    start = time.perf_counter()
+
+    while queue:
+        fn = queue.popleft()
+        if fn in visited:
+            continue
+        t0 = time.perf_counter()
+        visited.add(fn)
+        calls = graph.get(fn, [])
+        facts += len(calls)
+        for callee in calls:
+            if callee not in visited:
+                queue.append(callee)
+        timings_ms.append((time.perf_counter() - t0) * 1000.0)
+        cache_records.append(
+            {
+                "function": fn,
+                "reachable": True,
+                "runtimeCount": runtime.get(fn, 0),
+                "fanout": len(calls),
+            }
+        )
+
+    duration_ms = (time.perf_counter() - start) * 1000.0
+    _, peak_bytes = tracemalloc.get_traced_memory()
+    tracemalloc.stop()
+
+    if hasattr(gc, "get_stats"):
+        gc_stats = gc.get_stats()
+        gc_gen2 = gc_stats[2]["collections"] if len(gc_stats) > 2 else 0
+    else:
+        counts = gc.get_count()
+        gc_gen2 = counts[2] if len(counts) > 2 else 0
+
+    metrics = {
+        "run": "reachability",
+        "startedAtUtc": started_at,
+        "functions": len(graph),
+        "runtimeEvents": len(runtime),
+        "facts": facts,
+        "durationMs": round(duration_ms, 3),
+        "factsPerSec": round(facts / (duration_ms / 1000.0) if duration_ms else 0.0, 3),
+        "p50MsPerNode": round(percentile(timings_ms, 50), 4),
+        "p95MsPerNode": round(percentile(timings_ms, 95), 4),
+        "p99MsPerNode": round(percentile(timings_ms, 99), 4),
+        "rssMb": round(peak_bytes / (1024 * 1024), 3),
+        "managedMb": round(peak_bytes / (1024 * 1024), 3),
+        "gcGen2": gc_gen2,
+    }
+
+    return metrics, cache_records
+
+
+def write_ndjson(records: Iterable[dict], output: Path):
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with output.open("w", encoding="utf-8") as handle:
+        for record in records:
+            handle.write(json.dumps(record, separators=(",", ":"), sort_keys=True) + "\n")
+
+
+def parse_args(argv=None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Reachability scoring benchmark")
+    parser.add_argument("--callgraph", required=True, help="Path to callgraph NDJSON")
+    parser.add_argument("--runtime", required=True, help="Path to runtime NDJSON")
+    parser.add_argument("--output", default="results/reachability-metrics.ndjson", help="Output metrics NDJSON")
+    parser.add_argument("--cache-output", default="results/reachability-cache.ndjson", help="Cache output NDJSON")
+    parser.add_argument("--threads", type=int, default=1, help="Thread count (unused; for compatibility)")
+    parser.add_argument("--seed", type=int, default=20250101, help="Seed placeholder for deterministic behaviour")
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    callgraph_records = load_ndjson(Path(args.callgraph))
+    runtime_records = load_ndjson(Path(args.runtime))
+
+    graph = build_graph(callgraph_records)
+    runtime = build_runtime(runtime_records)
+
+    metrics, cache_records = run_reachability(graph, runtime)
+    write_ndjson([metrics], Path(args.output))
+    write_ndjson(cache_records, Path(args.cache_output))
+
+    print(
+        f"Wrote metrics to {args.output} and cache with {len(cache_records)} entries to {args.cache_output}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/Bench/StellaOps.Bench/Signals/results/reachability-cache-10k.ndjson.sha256

@@ -0,0 +1 @@
+415490c6ea6185c918a2205ad3a5bca99420d58da322084c6f61cbc1242fde2b  reachability-cache-10k.ndjson

src/Bench/StellaOps.Bench/Signals/results/reachability-cache-50k.ndjson.sha256

@@ -0,0 +1 @@
+a8d80bf1914e3b0a339520a9a2ba7b60718434ef0a7d44687f0d09ebe1ac5830  reachability-cache-50k.ndjson

src/Bench/StellaOps.Bench/Signals/results/reachability-metrics-10k.ndjson

@@ -0,0 +1 @@
+{"durationMs":58.532,"facts":30000,"factsPerSec":512540.149,"functions":10000,"gcGen2":1,"managedMb":2.66,"p50MsPerNode":0.0031,"p95MsPerNode":0.0036,"p99MsPerNode":0.0062,"rssMb":2.66,"run":"reachability","runtimeEvents":1000,"startedAtUtc":"2025-12-11T20:52:24.336490+00:00"}

src/Bench/StellaOps.Bench/Signals/results/reachability-metrics-10k.ndjson.sha256

@@ -0,0 +1 @@
+c6bf61890712d802b3ad288446b4754396774dfaa0d1e8502dc01ba6e8391cd0  reachability-metrics-10k.ndjson

src/Bench/StellaOps.Bench/Signals/results/reachability-metrics-50k.ndjson

@@ -0,0 +1 @@
+{"durationMs":304.323,"facts":150000,"factsPerSec":492897.349,"functions":50000,"gcGen2":1,"managedMb":12.811,"p50MsPerNode":0.0033,"p95MsPerNode":0.004,"p99MsPerNode":0.007,"rssMb":12.811,"run":"reachability","runtimeEvents":5000,"startedAtUtc":"2025-12-11T20:52:33.262306+00:00"}

src/Bench/StellaOps.Bench/Signals/results/reachability-metrics-50k.ndjson.sha256

@@ -0,0 +1 @@
+7686b8ffef892a6fa6e207a8a051786facbfa085e713ea3c717a7c2ae8ade97c  reachability-metrics-50k.ndjson

src/Bench/StellaOps.Bench/Signals/tests/__init__.py

@@ -0,0 +1 @@
+# Package marker for reachability bench tests.

src/Bench/StellaOps.Bench/Signals/tests/test_reachability_bench.py

@@ -0,0 +1,52 @@
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+import reachability_bench as bench
+
+
+class ReachabilityBenchTests(unittest.TestCase):
+    def test_reachability_metrics_and_cache(self):
+        tmp_dir = Path(tempfile.mkdtemp(prefix="reachability-bench-test-"))
+        callgraph_path = tmp_dir / "callgraph.ndjson"
+        runtime_path = tmp_dir / "runtime.ndjson"
+
+        callgraph_records = [
+            {"function": "fn-A", "calls": ["fn-B"], "weight": 1},
+            {"function": "fn-B", "calls": ["fn-C"], "weight": 1},
+            {"function": "fn-C", "calls": [], "weight": 1},
+        ]
+        runtime_records = [
+            {"function": "fn-A", "count": 2, "timestamp": "2025-01-01T00:00:00Z"},
+        ]
+
+        with callgraph_path.open("w", encoding="utf-8") as handle:
+            for rec in callgraph_records:
+                handle.write(json.dumps(rec) + "\n")
+        with runtime_path.open("w", encoding="utf-8") as handle:
+            for rec in runtime_records:
+                handle.write(json.dumps(rec) + "\n")
+
+        graph = bench.build_graph(bench.load_ndjson(callgraph_path))
+        runtime = bench.build_runtime(bench.load_ndjson(runtime_path))
+
+        metrics, cache = bench.run_reachability(graph, runtime)
+        self.assertEqual(metrics["functions"], 3)
+        self.assertEqual(metrics["runtimeEvents"], 1)
+        self.assertEqual(metrics["facts"], 2)
+        self.assertEqual(len(cache), 3)
+
+    def test_percentile(self):
+        values = [1, 2, 3]
+        self.assertAlmostEqual(bench.percentile(values, 50), 2.0)
+        self.assertAlmostEqual(bench.percentile(values, 99), 2.98, places=2)
+
+
+if __name__ == "__main__":
+    unittest.main()