up

2025-11-29 11:37:00 +02:00
parent 3488b22c0c
commit 8f54ffa203
14 changed files with 685 additions and 4 deletions
--- a/bench/reachability-benchmark/schemas/submission.schema.json
+++ b/bench/reachability-benchmark/schemas/submission.schema.json
@@ -0,0 +1,104 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://stellaops.org/benchmark/reachability/submission.schema.json",
+  "title": "Reachability Benchmark Submission",
+  "type": "object",
+  "required": ["version", "tool", "run", "cases"],
+  "additionalProperties": false,
+  "properties": {
+    "version": {
+      "type": "string",
+      "enum": ["1.0.0"],
+      "description": "Submission schema version"
+    },
+    "tool": {
+      "type": "object",
+      "required": ["name", "version"],
+      "additionalProperties": false,
+      "properties": {
+        "name": {"type": "string"},
+        "version": {"type": "string"}
+      }
+    },
+    "run": {
+      "type": "object",
+      "required": ["platform"],
+      "additionalProperties": false,
+      "description": "Execution metadata for reproducibility",
+      "properties": {
+        "commit": {"type": "string"},
+        "platform": {"type": "string"},
+        "time_s": {"type": "number", "minimum": 0},
+        "peak_mb": {"type": "number", "minimum": 0}
+      }
+    },
+    "cases": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": ["case_id", "sinks"],
+        "additionalProperties": false,
+        "properties": {
+          "case_id": {"type": "string"},
+          "sinks": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "object",
+              "required": ["sink_id", "prediction"],
+              "additionalProperties": false,
+              "properties": {
+                "sink_id": {"type": "string"},
+                "prediction": {
+                  "type": "string",
+                  "enum": ["reachable", "unreachable"]
+                },
+                "confidence": {
+                  "type": "number",
+                  "minimum": 0,
+                  "maximum": 1
+                },
+                "explain": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                    "entry": {"type": "string"},
+                    "path": {
+                      "type": "array",
+                      "items": {"type": "string"},
+                      "minItems": 2
+                    },
+                    "guards": {
+                      "type": "array",
+                      "items": {"type": "string"},
+                      "uniqueItems": true
+                    }
+                  }
+                },
+                "notes": {"type": "string"}
+              }
+            }
+          }
+        }
+      }
+    },
+    "artifacts": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "sbom": {"type": "string"},
+        "attestation": {"type": "string"}
+      }
+    },
+    "submitter": {
+      "type": "object",
+      "properties": {
+        "name": {"type": "string"},
+        "organization": {"type": "string"},
+        "contact": {"type": "string", "format": "email"}
+      },
+      "additionalProperties": false
+    }
+  }
+}