feat: Add JSON schema definitions for coverage and trace artifacts in reachability benchmark

This commit is contained in:
StellaOps Bot
2025-12-03 09:49:59 +02:00
parent de53785176
commit e0b585c799
3 changed files with 98 additions and 0 deletions

View File

@@ -3,8 +3,11 @@
- `case.schema.yaml` — case descriptor (language, sinks, deterministic build/test, environment, optional inline ground truth summary). - `case.schema.yaml` — case descriptor (language, sinks, deterministic build/test, environment, optional inline ground truth summary).
- `entrypoints.schema.yaml` — declared entrypoints grouped by type (`http`, `cli`, `scheduled`, etc.). - `entrypoints.schema.yaml` — declared entrypoints grouped by type (`http`, `cli`, `scheduled`, etc.).
- `truth.schema.json` — ground-truth labels + evidence per sink (`reachable`/`unreachable`/`unknown`). - `truth.schema.json` — ground-truth labels + evidence per sink (`reachable`/`unreachable`/`unknown`).
- `coverage.schema.json` — coverage artifact emitted by oracle tests (lines covered or function-level coverage).
- `trace.schema.json` — dynamic trace artifact (structured path or event log) proving reachability/unreachability.
- `submission.schema.json` — submission format (tool/run metadata, per-sink predictions, explanations). - `submission.schema.json` — submission format (tool/run metadata, per-sink predictions, explanations).
- `examples/` — minimal valid examples for each schema. - `examples/` — minimal valid examples for each schema.
- `benchmark/schemas/benchmark-manifest.schema.json` — kit manifest with hashed artifacts, resource limits, and optional DSSE signatures.
## Validate quickly ## Validate quickly
```bash ```bash
@@ -15,6 +18,8 @@ python -m pip install -r ../tools/requirements.txt
python ../tools/validate.py case examples/case.sample.yaml python ../tools/validate.py case examples/case.sample.yaml
python ../tools/validate.py entrypoints examples/entrypoints.sample.yaml python ../tools/validate.py entrypoints examples/entrypoints.sample.yaml
python ../tools/validate.py truth examples/truth.sample.json python ../tools/validate.py truth examples/truth.sample.json
python ../tools/validate.py coverage ../cases/js/unsafe-eval/outputs/coverage.json
python ../tools/validate.py trace ../cases/js/unsafe-eval/outputs/traces/traces.json
python ../tools/validate.py submission examples/submission.sample.json python ../tools/validate.py submission examples/submission.sample.json
# or validate everything in one shot # or validate everything in one shot

View File

@@ -0,0 +1,51 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stellaops.org/benchmark/reachability/coverage.schema.json",
"title": "Reachability Benchmark Coverage Artifact",
"type": "object",
"oneOf": [
{
"required": ["files"],
"properties": {
"files": {
"type": "object",
"additionalProperties": {
"type": "object",
"required": ["lines_covered", "lines_total"],
"properties": {
"lines_covered": {
"type": "array",
"items": { "type": "integer", "minimum": 1 }
},
"lines_total": { "type": "integer", "minimum": 1 }
},
"additionalProperties": false
}
}
},
"additionalProperties": false
},
{
"required": ["files"],
"properties": {
"files": {
"type": "array",
"items": {
"type": "object",
"required": ["path", "coverage"],
"properties": {
"path": { "type": "string" },
"functions": {
"type": "array",
"items": { "type": "string" }
},
"coverage": { "type": "number", "minimum": 0, "maximum": 1 }
},
"additionalProperties": false
}
}
},
"additionalProperties": false
}
]
}

View File

@@ -0,0 +1,42 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://stellaops.org/benchmark/reachability/trace.schema.json",
"title": "Reachability Benchmark Trace Artifact",
"type": "object",
"oneOf": [
{
"required": ["entry", "path", "sink"],
"properties": {
"entry": { "type": "string" },
"path": {
"type": "array",
"minItems": 1,
"items": { "type": "string" }
},
"sink": { "type": "string" },
"notes": { "type": "string" }
},
"additionalProperties": false
},
{
"required": ["events"],
"properties": {
"events": {
"type": "array",
"items": {
"type": "object",
"required": ["path", "type"],
"properties": {
"path": { "type": "string" },
"type": { "type": "string" },
"at": { "type": "integer", "minimum": 0 }
},
"additionalProperties": false
}
},
"notes": { "type": "string" }
},
"additionalProperties": false
}
]
}