Add receipt input JSON and SHA256 hash for CVSS policy scoring tests

- Introduced a new JSON fixture `receipt-input.json` containing base, environmental, and threat metrics for CVSS scoring. - Added corresponding SHA256 hash file `receipt-input.sha256` to ensure integrity of the JSON fixture.
2025-12-04 07:30:42 +02:00
parent 2d079d61ed
commit e1262eb916
91 changed files with 19493 additions and 187 deletions
--- a/bench/reachability-benchmark/benchmark/CHANGELOG.md
+++ b/bench/reachability-benchmark/benchmark/CHANGELOG.md
@@ -0,0 +1,11 @@
+# Reachability Benchmark Changelog
+
+## 1.0.1 · 2025-12-03
+- Added manifest schema + sample manifest with hashes, SBOM/attestation entries, and sandbox/redaction metadata.
+- Added coverage/trace schemas and extended validator to cover them.
+- Introduced `tools/verify_manifest.py` and deterministic offline kit packaging script.
+- Added per-language determinism env templates and dataset safety checklist.
+- Populated SBOM + attestation outputs for JS/PY/C tracks; Java remains blocked on JDK availability.
+
+## 1.0.0 · 2025-12-01
+- Initial public dataset, scorer, baselines, and website.
--- a/bench/reachability-benchmark/benchmark/checklists/dataset-safety.md
+++ b/bench/reachability-benchmark/benchmark/checklists/dataset-safety.md
@@ -0,0 +1,15 @@
+# Dataset Safety & Provenance Checklist (RD1–RD10)
+
+Version: 1.0.1 · Date: 2025-12-03
+
+- [x] PII/secret scrub: no tokens/URLs; build/test logs redacted. Attested by DSSE when signing manifest.
+- [x] License compatibility: all cases authored in-repo under Apache-2.0; third-party snippets none. NOTICE up to date.
+- [x] Feed/tool lockfile: manifest.sample.json pins hashes for schemas, scorer, builder, and baseline submissions (when present).
+- [x] Published schemas/validators: truth/submission/coverage/trace + manifest schemas; validated via `tools/validate.py` and `tools/verify_manifest.py`.
+- [x] Evidence bundles: coverage + traces + attestation + sbom recorded per case (sample manifest).
+- [x] Binary case recipe: `cases/**/build/build.sh` pinned `SOURCE_DATE_EPOCH` and env templates under `benchmark/templates/determinism/`.
+- [x] Determinism CI: `ci/run-ci.sh` + `tools/verify_manifest.py` run twice to compare hashes; Java track still blocked on JDK availability.
+- [x] Signed baselines: baseline submissions may include DSSE path in manifest (not required for sample kit); rulepack hashes recorded separately.
+- [x] Submission policy: CLA/DSSE optional in sample; production kits require DSSE envelope recorded in `signatures`.
+- [x] Semantic versioning & changelog: see `benchmark/CHANGELOG.md`; manifest `version` mirrors dataset release.
+- [x] Offline kit packaging: `tools/package_offline_kit.sh` produces deterministic tarball with manifest + schemas + tools.
--- a/bench/reachability-benchmark/benchmark/manifest.sample.json
+++ b/bench/reachability-benchmark/benchmark/manifest.sample.json
@@ -0,0 +1,92 @@
+{
+  "schemaVersion": "1.0.0",
+  "kitId": "reachability-benchmark:public-v1",
+  "version": "1.0.1",
+  "createdAt": "2025-12-03T00:00:00Z",
+  "sourceDateEpoch": 1730000000,
+  "resourceLimits": {
+    "cpu": "4",
+    "memory": "8Gi"
+  },
+  "cases": [
+    {
+      "id": "js-unsafe-eval:001",
+      "language": "js",
+      "size": "small",
+      "hashes": {
+        "source": { "path": "cases/js/unsafe-eval", "sha256": "69b0d1cbae1e2c9ddc0f4dba8c6db507e1d3a1c5ea0a0a545c6f3e785529c91c" },
+        "case": { "path": "cases/js/unsafe-eval/case.yaml", "sha256": "a858ff509fda65d69df476e870d9646c6a84744010c812f3d23a88576f20cb6b" },
+        "entrypoints": { "path": "cases/js/unsafe-eval/entrypoints.yaml", "sha256": "77829e728d34c9dc5f56c04784c97f619830ad43bd8410acb3d7134f372a49b3" },
+        "binary": { "path": "cases/js/unsafe-eval/outputs/binary.tar.gz", "sha256": "72da19f28c2c36b6666afcc304514b387de20a5de881d5341067481e8418e23e" },
+        "sbom": { "path": "cases/js/unsafe-eval/outputs/sbom.cdx.json", "sha256": "c00ee1e12b1b6a6237e42174b2fe1393bcf575f6605205a2b84366e867b36d5f" },
+        "coverage": { "path": "cases/js/unsafe-eval/outputs/coverage.json", "sha256": "c2cf5af508d33f6ecdc7c0f10200a02a4c0ddeb8e1fc08b55d9bd4a2d6cb926b" },
+        "traces": { "path": "cases/js/unsafe-eval/outputs/traces/traces.json", "sha256": "6e63c78e091cc9d06acdc5966dd9e54593ca6b0b97f502928de278b3f80adbd8" },
+        "attestation": { "path": "cases/js/unsafe-eval/outputs/attestation.json", "sha256": "be3b0971d805f68730a1c4c0f7a4c3c40dfc7a73099a5524c68759fcc1729d7c" },
+        "truth": { "path": "benchmark/truth/js-unsafe-eval.json", "sha256": "ab42f28ed229eb657ffcb36c3a99287436e1822a4c7d395a94de784457a08f62" }
+      },
+      "truth": {
+        "label": "reachable",
+        "confidence": "high",
+        "rationale": "Unit test hits eval sink via POST /api/exec"
+      },
+      "sandbox": { "network": "loopback", "privileges": "rootless" },
+      "redaction": { "pii": false, "policy": "benchmark-default/v1" }
+    },
+    {
+      "id": "py-fastapi-guarded:104",
+      "language": "py",
+      "size": "small",
+      "hashes": {
+        "source": { "path": "cases/py/fastapi-guarded", "sha256": "0869cab10767ac7e7b33c9bbd634f811d98ce5cdeb244769f1a81949438460fb" },
+        "case": { "path": "cases/py/fastapi-guarded/case.yaml", "sha256": "0add8a5f487ebd21ee20ab88b7c6436fe8471f0a54ab8da0e08c8416aa181346" },
+        "entrypoints": { "path": "cases/py/fastapi-guarded/entrypoints.yaml", "sha256": "47c9dd15bf7c5bb8641893a92791d3f7675ed6adba17b251f609335400d29d41" },
+        "binary": { "path": "cases/py/fastapi-guarded/outputs/binary.tar.gz", "sha256": "ca964fef352dc535b63d35b8f8846cc051e10e54cfd8aceef7566f3c94178b76" },
+        "sbom": { "path": "cases/py/fastapi-guarded/outputs/sbom.cdx.json", "sha256": "13999d8f3d4c9bdb70ea54ad1de613be3f893d79bdd1a53f7c9401e6add88cf0" },
+        "coverage": { "path": "cases/py/fastapi-guarded/outputs/coverage.json", "sha256": "07b1f6dccaa02bd4e1c3e2771064fa3c6e06d02843a724151721ea694762c750" },
+        "traces": { "path": "cases/py/fastapi-guarded/outputs/traces/traces.json", "sha256": "4633748b8b428b45e3702f2f8f5b3f4270728078e26bce1e08900ed1d5bb3046" },
+        "attestation": { "path": "cases/py/fastapi-guarded/outputs/attestation.json", "sha256": "257aa5408a5c6ffe0e193a75a2a54597f8c6f61babfe8aaf26bd47340c3086c3" },
+        "truth": { "path": "benchmark/truth/py-fastapi-guarded.json", "sha256": "f8c62abeb00006621feeb010d0e47d248918dffd6d6e20e0f47d74e1b3642760" }
+      },
+      "truth": {
+        "label": "unreachable",
+        "confidence": "high",
+        "rationale": "Feature flag ALLOW_EXEC must be true before sink executes"
+      },
+      "sandbox": { "network": "loopback", "privileges": "rootless" },
+      "redaction": { "pii": false, "policy": "benchmark-default/v1" }
+    },
+    {
+      "id": "c-unsafe-system:001",
+      "language": "c",
+      "size": "small",
+      "hashes": {
+        "source": { "path": "cases/c/unsafe-system", "sha256": "bc39ab3a3e5cb3944a205912ecad8c1ac4b7d15c64b453c9d34a9a5df7fbbbf4" },
+        "case": { "path": "cases/c/unsafe-system/case.yaml", "sha256": "7799a3a629c22ad47197309f44e32aabbc4e6711ef78d606ba57a7a4974787ce" },
+        "entrypoints": { "path": "cases/c/unsafe-system/entrypoints.yaml", "sha256": "06afee8350460c9d15b26ea9d4ea293e8eb3f4b86b3179e19401fa99947e4490" },
+        "binary": { "path": "cases/c/unsafe-system/outputs/binary.tar.gz", "sha256": "62200167bd660bad6d131b21f941acdfebe00e949e353a53c97b6691ac8f0e49" },
+        "sbom": { "path": "cases/c/unsafe-system/outputs/sbom.cdx.json", "sha256": "4c72a213fc4c646f44b4d0be3c23711b120b2a386374ebaa4897e5058980e0f5" },
+        "coverage": { "path": "cases/c/unsafe-system/outputs/coverage.json", "sha256": "03ba8cf09e7e0ed82e9fa8abb48f92355e894fd56e0c0160a504193a6f6ec48a" },
+        "traces": { "path": "cases/c/unsafe-system/outputs/traces/traces.json", "sha256": "f6469e46a57b8a6e8e17c9b8e78168edd6657ea8a5e1e96fe6ab4a0fc88a734e" },
+        "attestation": { "path": "cases/c/unsafe-system/outputs/attestation.json", "sha256": "c3755088182359a45492170fa8a57d826b605176333d109f4f113bc7ccf85f97" },
+        "truth": { "path": "benchmark/truth/c-unsafe-system.json", "sha256": "9a8200c2cf549b3ac8b19b170e9d34df063351879f19f401d8492e280ad08c13" }
+      },
+      "truth": {
+        "label": "reachable",
+        "confidence": "high",
+        "rationale": "Command injection sink reachable via argv -> system()"
+      },
+      "sandbox": { "network": "loopback", "privileges": "rootless" },
+      "redaction": { "pii": false, "policy": "benchmark-default/v1" }
+    }
+  ],
+  "artifacts": {
+    "submissionSchema": { "path": "schemas/submission.schema.json", "sha256": "de5bebb2dbcd085d7896f47a16b9d3837a65fb7f816dcf7e587967d5848c50a7" },
+    "scorer": { "path": "tools/scorer/rb_score.py", "sha256": "32d4f69f5d1d4b87902d6c4f020efde703487d526bf7d42b4438cb2499813f7f" },
+    "baselineSubmissions": []
+  },
+  "tools": {
+    "builder": { "path": "tools/build/build_all.py", "sha256": "64a73f3df9b6f2cdaf5cbb33852b8e9bf443f67cf9dff1573fb635a0252bda9a" },
+    "validator": { "path": "tools/validate.py", "sha256": "776009ef0f3691e60cc87df3f0468181ee7a827be1bd0f73c77fdb68d3ed31c0" }
+  },
+  "signatures": []
+}
--- a/bench/reachability-benchmark/benchmark/templates/determinism/c.env
+++ b/bench/reachability-benchmark/benchmark/templates/determinism/c.env
@@ -0,0 +1,5 @@
+# Deterministic defaults for C cases
+SOURCE_DATE_EPOCH=1730000000
+TZ=UTC
+LANG=C
+LC_ALL=C
--- a/bench/reachability-benchmark/benchmark/templates/determinism/java.env
+++ b/bench/reachability-benchmark/benchmark/templates/determinism/java.env
@@ -0,0 +1,3 @@
+# Deterministic defaults for Java cases
+SOURCE_DATE_EPOCH=1730000000
+JAVA_TOOL_OPTIONS="-Duser.country=US -Duser.language=en -Duser.timezone=UTC -Dfile.encoding=UTF-8"
--- a/bench/reachability-benchmark/benchmark/templates/determinism/js.env
+++ b/bench/reachability-benchmark/benchmark/templates/determinism/js.env
@@ -0,0 +1,5 @@
+# Deterministic defaults for JavaScript/Node cases
+SOURCE_DATE_EPOCH=1730000000
+NODE_OPTIONS=--no-deprecation
+NPM_CONFIG_FUND=false
+NPM_CONFIG_AUDIT=false
--- a/bench/reachability-benchmark/benchmark/templates/determinism/py.env
+++ b/bench/reachability-benchmark/benchmark/templates/determinism/py.env
@@ -0,0 +1,5 @@
+# Deterministic defaults for Python cases
+SOURCE_DATE_EPOCH=1730000000
+PYTHONHASHSEED=1
+PIP_DISABLE_PIP_VERSION_CHECK=1
+PYTHONUTF8=1