feat: Add JSON schema definitions for coverage and trace artifacts in reachability benchmark
This commit is contained in:
@@ -3,8 +3,11 @@
|
|||||||
- `case.schema.yaml` — case descriptor (language, sinks, deterministic build/test, environment, optional inline ground truth summary).
|
- `case.schema.yaml` — case descriptor (language, sinks, deterministic build/test, environment, optional inline ground truth summary).
|
||||||
- `entrypoints.schema.yaml` — declared entrypoints grouped by type (`http`, `cli`, `scheduled`, etc.).
|
- `entrypoints.schema.yaml` — declared entrypoints grouped by type (`http`, `cli`, `scheduled`, etc.).
|
||||||
- `truth.schema.json` — ground-truth labels + evidence per sink (`reachable`/`unreachable`/`unknown`).
|
- `truth.schema.json` — ground-truth labels + evidence per sink (`reachable`/`unreachable`/`unknown`).
|
||||||
|
- `coverage.schema.json` — coverage artifact emitted by oracle tests (lines covered or function-level coverage).
|
||||||
|
- `trace.schema.json` — dynamic trace artifact (structured path or event log) proving reachability/unreachability.
|
||||||
- `submission.schema.json` — submission format (tool/run metadata, per-sink predictions, explanations).
|
- `submission.schema.json` — submission format (tool/run metadata, per-sink predictions, explanations).
|
||||||
- `examples/` — minimal valid examples for each schema.
|
- `examples/` — minimal valid examples for each schema.
|
||||||
|
- `benchmark/schemas/benchmark-manifest.schema.json` — kit manifest with hashed artifacts, resource limits, and optional DSSE signatures.
|
||||||
|
|
||||||
## Validate quickly
|
## Validate quickly
|
||||||
```bash
|
```bash
|
||||||
@@ -15,6 +18,8 @@ python -m pip install -r ../tools/requirements.txt
|
|||||||
python ../tools/validate.py case examples/case.sample.yaml
|
python ../tools/validate.py case examples/case.sample.yaml
|
||||||
python ../tools/validate.py entrypoints examples/entrypoints.sample.yaml
|
python ../tools/validate.py entrypoints examples/entrypoints.sample.yaml
|
||||||
python ../tools/validate.py truth examples/truth.sample.json
|
python ../tools/validate.py truth examples/truth.sample.json
|
||||||
|
python ../tools/validate.py coverage ../cases/js/unsafe-eval/outputs/coverage.json
|
||||||
|
python ../tools/validate.py trace ../cases/js/unsafe-eval/outputs/traces/traces.json
|
||||||
python ../tools/validate.py submission examples/submission.sample.json
|
python ../tools/validate.py submission examples/submission.sample.json
|
||||||
|
|
||||||
# or validate everything in one shot
|
# or validate everything in one shot
|
||||||
|
|||||||
51
bench/reachability-benchmark/schemas/coverage.schema.json
Normal file
51
bench/reachability-benchmark/schemas/coverage.schema.json
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://stellaops.org/benchmark/reachability/coverage.schema.json",
|
||||||
|
"title": "Reachability Benchmark Coverage Artifact",
|
||||||
|
"type": "object",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"required": ["files"],
|
||||||
|
"properties": {
|
||||||
|
"files": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["lines_covered", "lines_total"],
|
||||||
|
"properties": {
|
||||||
|
"lines_covered": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "integer", "minimum": 1 }
|
||||||
|
},
|
||||||
|
"lines_total": { "type": "integer", "minimum": 1 }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"required": ["files"],
|
||||||
|
"properties": {
|
||||||
|
"files": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["path", "coverage"],
|
||||||
|
"properties": {
|
||||||
|
"path": { "type": "string" },
|
||||||
|
"functions": {
|
||||||
|
"type": "array",
|
||||||
|
"items": { "type": "string" }
|
||||||
|
},
|
||||||
|
"coverage": { "type": "number", "minimum": 0, "maximum": 1 }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
42
bench/reachability-benchmark/schemas/trace.schema.json
Normal file
42
bench/reachability-benchmark/schemas/trace.schema.json
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://stellaops.org/benchmark/reachability/trace.schema.json",
|
||||||
|
"title": "Reachability Benchmark Trace Artifact",
|
||||||
|
"type": "object",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"required": ["entry", "path", "sink"],
|
||||||
|
"properties": {
|
||||||
|
"entry": { "type": "string" },
|
||||||
|
"path": {
|
||||||
|
"type": "array",
|
||||||
|
"minItems": 1,
|
||||||
|
"items": { "type": "string" }
|
||||||
|
},
|
||||||
|
"sink": { "type": "string" },
|
||||||
|
"notes": { "type": "string" }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"required": ["events"],
|
||||||
|
"properties": {
|
||||||
|
"events": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "object",
|
||||||
|
"required": ["path", "type"],
|
||||||
|
"properties": {
|
||||||
|
"path": { "type": "string" },
|
||||||
|
"type": { "type": "string" },
|
||||||
|
"at": { "type": "integer", "minimum": 0 }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notes": { "type": "string" }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user