Add tests and implement timeline ingestion options with NATS and Redis subscribers
- Introduced `BinaryReachabilityLifterTests` to validate binary lifting functionality. - Created `PackRunWorkerOptions` for configuring worker paths and execution persistence. - Added `TimelineIngestionOptions` for configuring NATS and Redis ingestion transports. - Implemented `NatsTimelineEventSubscriber` for subscribing to NATS events. - Developed `RedisTimelineEventSubscriber` for reading from Redis Streams. - Added `TimelineEnvelopeParser` to normalize incoming event envelopes. - Created unit tests for `TimelineEnvelopeParser` to ensure correct field mapping. - Implemented `TimelineAuthorizationAuditSink` for logging authorization outcomes.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,43 @@
|
||||
# Deterministic SBOM composition fixtures
|
||||
|
||||
Reference bundle for DOCS-SCANNER-DET-01. Use it to prove fragment-level DSSE, `_composition.json`, and CycloneDX composition metadata stay deterministic and offline-verifiable.
|
||||
|
||||
## Contents
|
||||
- `_composition.json` — composition recipe with Merkle root, fragment hashes, BOM hash, and determinism pins.
|
||||
- `fragment-layer{1,2}.json` — canonical fragments (sorted keys, newline-terminated).
|
||||
- `fragment-layer{1,2}.dsse.json` — DSSE envelopes over the canonical fragments (demo key `demo-ed25519`).
|
||||
- `bom.cdx.json` — composed CycloneDX BOM with `stellaops:merkle.root` and `stellaops:composition.manifest` properties.
|
||||
- `hashes.txt` — sha256 for every file in this directory.
|
||||
- `generate.py` — reproducible generator (standard library only).
|
||||
|
||||
## Verify offline
|
||||
```bash
|
||||
cd docs/modules/scanner/fixtures/deterministic-compose
|
||||
sha256sum -c hashes.txt
|
||||
|
||||
# Check DSSE payload matches fragment
|
||||
jq -r '.payload' fragment-layer1.dsse.json | base64 -d > /tmp/payload.json
|
||||
diff -u fragment-layer1.json /tmp/payload.json
|
||||
|
||||
# Recompute Merkle root from fragment hashes
|
||||
python - <<'PY'
|
||||
import hashlib, json
|
||||
from pathlib import Path
|
||||
frag_hashes = [line.split()[0] for line in Path('hashes.txt').read_text().splitlines() if 'fragment-layer' in line and '.json' in line and '.dsse' not in line]
|
||||
frag_hashes = [bytes.fromhex(h) for h in frag_hashes]
|
||||
while len(frag_hashes) > 1:
|
||||
nxt = []
|
||||
it = iter(frag_hashes)
|
||||
for a in it:
|
||||
b = next(it, a)
|
||||
nxt.append(hashlib.sha256(a+b).digest())
|
||||
frag_hashes = nxt
|
||||
print(f"merkle={frag_hashes[0].hex()}")
|
||||
PY
|
||||
```
|
||||
|
||||
## Regenerate
|
||||
```bash
|
||||
python generate.py
|
||||
sha256sum -c hashes.txt
|
||||
```
|
||||
@@ -0,0 +1 @@
|
||||
{"composedBomSha256":"c161ac9cfee5f3baee69d303a0fb70bfb036d863e317e6e0d5843b983a6c8466","determinism":{"feedSnapshotId":"feeds-2025.320.1","fixedClock":"2025-12-01T00:00:00Z","policySnapshotId":"policy-2025.310.0","rngSeed":1337},"fragments":[{"dsseEnvelopeSha256":"ff008ab332bbcc6ac413739eb66529c3fcb1ca2d2503f8263bf5e0645d930118","dssePath":"fragment-layer1.dsse.json","fragmentPath":"fragment-layer1.json","fragmentSha256":"7884ea6f3a46a0870d8fc74a5e770bac49a9729a83175dbcf42ca14769b22fa0","layerDigest":"sha256:1111111111111111111111111111111111111111111111111111111111111101"},{"dsseEnvelopeSha256":"8813b84f072196808e644e6a8c54a81348b566054149b26a0055d8e63e0ae6aa","dssePath":"fragment-layer2.dsse.json","fragmentPath":"fragment-layer2.json","fragmentSha256":"cb9783249cf18e8d8a227d288864d821c190005897a14212f21742c0f404208f","layerDigest":"sha256:2222222222222222222222222222222222222222222222222222222222222202"}],"generatedAtUtc":"2025-12-03T00:00:00Z","imageDigest":"sha256:9999999999999999999999999999999999999999999999999999999999999900","merkleRootSha256":"963e421d21be2db87895ea5fd973a0ad71aa638499c274308e013d2b6c8243f6","schemaVersion":"1.0"}
|
||||
@@ -0,0 +1 @@
|
||||
{"bomFormat":"CycloneDX","components":[{"bom-ref":"pkg:apk/alpine/busybox@1.36.1","name":"busybox","properties":[{"name":"stellaops:stella.contentHash","value":"7884ea6f3a46a0870d8fc74a5e770bac49a9729a83175dbcf42ca14769b22fa0"}],"purl":"pkg:apk/alpine/busybox@1.36.1","type":"library","version":"1.36.1"},{"bom-ref":"pkg:npm/express@4.18.2","name":"express","properties":[{"name":"stellaops:stella.contentHash","value":"cb9783249cf18e8d8a227d288864d821c190005897a14212f21742c0f404208f"}],"purl":"pkg:npm/express@4.18.2","type":"library","version":"4.18.2"}],"metadata":{"component":{"bom-ref":"pkg:docker/registry.local/demo@sha256:9999999999999999999999999999999999999999999999999999999999999900","name":"registry.local/demo","purl":"pkg:docker/registry.local/demo@sha256:9999999999999999999999999999999999999999999999999999999999999900","type":"container"},"timestamp":"2025-12-03T00:00:00Z"},"properties":[{"name":"stellaops:merkle.root","value":"963e421d21be2db87895ea5fd973a0ad71aa638499c274308e013d2b6c8243f6"},{"name":"stellaops:composition.manifest","value":"cas://scanner/deterministic-compose/_composition.json"},{"name":"stellaops:stella.contentHash","value":"963e421d21be2db87895ea5fd973a0ad71aa638499c274308e013d2b6c8243f6"}],"serialNumber":"urn:uuid:00000000-7e57-4c0d-baad-000000000301","specVersion":"1.6","version":1}
|
||||
@@ -0,0 +1 @@
|
||||
{"payload":"eyJjb21wb25lbnRzIjpbeyJldmlkZW5jZSI6eyJjb250ZW50SGFzaCI6InNoYTI1Njo3YzNmNGQzMGJmY2Q4ZmYyYjA5ZjFiYTM5ZjQzYzUyNGQ2Y2UxYjdhNWYzYzJiZGUzMjFlMGY1ZTBlNmMzZDEwIiwibGljZW5zZXMiOlsiQlNELTMtQ2xhdXNlIl0sInBhdGhzIjpbIi9iaW4vYnVzeWJveCJdfSwiaWRlbnRpdHkiOnsicHVybCI6InBrZzphcGsvYWxwaW5lL2J1c3lib3hAMS4zNi4xIn0sInNvdXJjZSI6ImFwayJ9XSwiZ2VuZXJhdGVkQXRVdGMiOiIyMDI1LTEyLTAxVDAwOjAwOjAwWiIsImxheWVyRGlnZXN0Ijoic2hhMjU2OjExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMTExMDEiLCJzY2hlbWFWZXJzaW9uIjoiMS4wIn0=","payloadType":"application/vnd.stellaops.scanner.fragment+json","signatures":[{"keyid":"demo-ed25519","sig":"ZGV0ZXJtaW5pc3RpYy1maXh0dXJlLWZyYWdtZW50LTE="}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"components":[{"evidence":{"contentHash":"sha256:7c3f4d30bfcd8ff2b09f1ba39f43c524d6ce1b7a5f3c2bde321e0f5e0e6c3d10","licenses":["BSD-3-Clause"],"paths":["/bin/busybox"]},"identity":{"purl":"pkg:apk/alpine/busybox@1.36.1"},"source":"apk"}],"generatedAtUtc":"2025-12-01T00:00:00Z","layerDigest":"sha256:1111111111111111111111111111111111111111111111111111111111111101","schemaVersion":"1.0"}
|
||||
@@ -0,0 +1 @@
|
||||
{"payload":"eyJjb21wb25lbnRzIjpbeyJldmlkZW5jZSI6eyJjb250ZW50SGFzaCI6InNoYTI1Njo4YWIxMDNmZWQ1OGU3ZGMwYjE4MTliNzM1ODEyNmQxYzQ0Y2M5NzlmNDA5Nzc1ODg4Yjg1OTUwNGE4MjkxNDhiIiwibGljZW5zZXMiOlsiTUlUIl0sInBhdGhzIjpbIi93b3Jrc3BhY2Uvbm9kZV9tb2R1bGVzL2V4cHJlc3MiXX0sImlkZW50aXR5Ijp7InB1cmwiOiJwa2c6bnBtL2V4cHJlc3NANC4xOC4yIn0sInNvdXJjZSI6Im5wbSJ9XSwiZ2VuZXJhdGVkQXRVdGMiOiIyMDI1LTEyLTAxVDAwOjAwOjAwWiIsImxheWVyRGlnZXN0Ijoic2hhMjU2OjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMDIiLCJzY2hlbWFWZXJzaW9uIjoiMS4wIn0=","payloadType":"application/vnd.stellaops.scanner.fragment+json","signatures":[{"keyid":"demo-ed25519","sig":"ZGV0ZXJtaW5pc3RpYy1maXh0dXJlLWZyYWdtZW50LTI="}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"components":[{"evidence":{"contentHash":"sha256:8ab103fed58e7dc0b1819b7358126d1c44cc979f409775888b859504a829148b","licenses":["MIT"],"paths":["/workspace/node_modules/express"]},"identity":{"purl":"pkg:npm/express@4.18.2"},"source":"npm"}],"generatedAtUtc":"2025-12-01T00:00:00Z","layerDigest":"sha256:2222222222222222222222222222222222222222222222222222222222222202","schemaVersion":"1.0"}
|
||||
190
docs/modules/scanner/fixtures/deterministic-compose/generate.py
Normal file
190
docs/modules/scanner/fixtures/deterministic-compose/generate.py
Normal file
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate deterministic SBOM composition fixtures.
|
||||
|
||||
Outputs fragment JSON, DSSE envelopes, a composition manifest, a composed
|
||||
CycloneDX BOM, and a hashes file suitable for offline verification.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent
|
||||
|
||||
|
||||
def canonical(obj) -> str:
|
||||
return json.dumps(obj, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def write_json(path: Path, obj) -> str:
|
||||
text = canonical(obj) + "\n"
|
||||
path.write_text(text)
|
||||
return hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def merkle_root(hex_hashes: list[str]) -> str:
|
||||
if not hex_hashes:
|
||||
return ""
|
||||
nodes = [bytes.fromhex(h) for h in hex_hashes]
|
||||
while len(nodes) > 1:
|
||||
nxt = []
|
||||
it = iter(nodes)
|
||||
for a in it:
|
||||
b = next(it, a)
|
||||
nxt.append(hashlib.sha256(a + b).digest())
|
||||
nodes = nxt
|
||||
return nodes[0].hex()
|
||||
|
||||
|
||||
def dsse_envelope(payload_json: str, label: str) -> dict:
|
||||
payload_b64 = base64.b64encode(payload_json.encode()).decode()
|
||||
signature = base64.b64encode(f"deterministic-fixture-{label}".encode()).decode()
|
||||
return {
|
||||
"payloadType": "application/vnd.stellaops.scanner.fragment+json",
|
||||
"payload": payload_b64,
|
||||
"signatures": [
|
||||
{
|
||||
"keyid": "demo-ed25519",
|
||||
"sig": signature,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ROOT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fragments_src = [
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"layerDigest": "sha256:1111111111111111111111111111111111111111111111111111111111111101",
|
||||
"generatedAtUtc": "2025-12-01T00:00:00Z",
|
||||
"components": [
|
||||
{
|
||||
"identity": {"purl": "pkg:apk/alpine/busybox@1.36.1"},
|
||||
"evidence": {
|
||||
"paths": ["/bin/busybox"],
|
||||
"licenses": ["BSD-3-Clause"],
|
||||
"contentHash": "sha256:7c3f4d30bfcd8ff2b09f1ba39f43c524d6ce1b7a5f3c2bde321e0f5e0e6c3d10",
|
||||
},
|
||||
"source": "apk",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"layerDigest": "sha256:2222222222222222222222222222222222222222222222222222222222222202",
|
||||
"generatedAtUtc": "2025-12-01T00:00:00Z",
|
||||
"components": [
|
||||
{
|
||||
"identity": {"purl": "pkg:npm/express@4.18.2"},
|
||||
"evidence": {
|
||||
"paths": ["/workspace/node_modules/express"],
|
||||
"licenses": ["MIT"],
|
||||
"contentHash": "sha256:8ab103fed58e7dc0b1819b7358126d1c44cc979f409775888b859504a829148b",
|
||||
},
|
||||
"source": "npm",
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
fragments_meta = []
|
||||
for idx, fragment in enumerate(fragments_src, start=1):
|
||||
fragment_path = ROOT / f"fragment-layer{idx}.json"
|
||||
fragment_hash = write_json(fragment_path, fragment)
|
||||
|
||||
envelope_obj = dsse_envelope(canonical(fragment), f"fragment-{idx}")
|
||||
envelope_path = ROOT / f"fragment-layer{idx}.dsse.json"
|
||||
envelope_hash = write_json(envelope_path, envelope_obj)
|
||||
|
||||
fragments_meta.append(
|
||||
{
|
||||
"layerDigest": fragment["layerDigest"],
|
||||
"fragmentPath": fragment_path.name,
|
||||
"dssePath": envelope_path.name,
|
||||
"fragmentSha256": fragment_hash,
|
||||
"dsseEnvelopeSha256": envelope_hash,
|
||||
}
|
||||
)
|
||||
|
||||
fragments_meta.sort(key=lambda f: f["layerDigest"])
|
||||
merkle = merkle_root([f["fragmentSha256"] for f in fragments_meta])
|
||||
|
||||
bom = {
|
||||
"bomFormat": "CycloneDX",
|
||||
"specVersion": "1.6",
|
||||
"serialNumber": "urn:uuid:00000000-7e57-4c0d-baad-000000000301",
|
||||
"version": 1,
|
||||
"metadata": {
|
||||
"timestamp": "2025-12-03T00:00:00Z",
|
||||
"component": {
|
||||
"type": "container",
|
||||
"bom-ref": "pkg:docker/registry.local/demo@sha256:9999999999999999999999999999999999999999999999999999999999999900",
|
||||
"name": "registry.local/demo",
|
||||
"purl": "pkg:docker/registry.local/demo@sha256:9999999999999999999999999999999999999999999999999999999999999900",
|
||||
},
|
||||
},
|
||||
"components": [
|
||||
{
|
||||
"bom-ref": "pkg:apk/alpine/busybox@1.36.1",
|
||||
"type": "library",
|
||||
"name": "busybox",
|
||||
"version": "1.36.1",
|
||||
"purl": "pkg:apk/alpine/busybox@1.36.1",
|
||||
"properties": [
|
||||
{"name": "stellaops:stella.contentHash", "value": fragments_meta[0]["fragmentSha256"]}
|
||||
],
|
||||
},
|
||||
{
|
||||
"bom-ref": "pkg:npm/express@4.18.2",
|
||||
"type": "library",
|
||||
"name": "express",
|
||||
"version": "4.18.2",
|
||||
"purl": "pkg:npm/express@4.18.2",
|
||||
"properties": [
|
||||
{"name": "stellaops:stella.contentHash", "value": fragments_meta[1]["fragmentSha256"]}
|
||||
],
|
||||
},
|
||||
],
|
||||
"properties": [
|
||||
{"name": "stellaops:merkle.root", "value": merkle},
|
||||
{"name": "stellaops:composition.manifest", "value": "cas://scanner/deterministic-compose/_composition.json"},
|
||||
{"name": "stellaops:stella.contentHash", "value": merkle},
|
||||
],
|
||||
}
|
||||
bom_path = ROOT / "bom.cdx.json"
|
||||
bom_hash = write_json(bom_path, bom)
|
||||
|
||||
composition = {
|
||||
"schemaVersion": "1.0",
|
||||
"imageDigest": "sha256:9999999999999999999999999999999999999999999999999999999999999900",
|
||||
"generatedAtUtc": "2025-12-03T00:00:00Z",
|
||||
"fragments": fragments_meta,
|
||||
"merkleRootSha256": merkle,
|
||||
"composedBomSha256": bom_hash,
|
||||
"determinism": {
|
||||
"fixedClock": "2025-12-01T00:00:00Z",
|
||||
"rngSeed": 1337,
|
||||
"feedSnapshotId": "feeds-2025.320.1",
|
||||
"policySnapshotId": "policy-2025.310.0",
|
||||
},
|
||||
}
|
||||
composition_path = ROOT / "_composition.json"
|
||||
composition_hash = write_json(composition_path, composition)
|
||||
|
||||
hashes = {
|
||||
"_composition.json": composition_hash,
|
||||
"bom.cdx.json": bom_hash,
|
||||
}
|
||||
for meta in fragments_meta:
|
||||
hashes[meta["fragmentPath"]] = meta["fragmentSha256"]
|
||||
hashes[meta["dssePath"]] = meta["dsseEnvelopeSha256"]
|
||||
|
||||
hashes_lines = [f"{hashes[name]} {name}" for name in sorted(hashes.keys())]
|
||||
(ROOT / "hashes.txt").write_text("\n".join(hashes_lines) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,6 @@
|
||||
c5c2e7195eb6f1624534966624655734fe595666f43a6b3dd168d60b5b33d5b0 _composition.json
|
||||
c161ac9cfee5f3baee69d303a0fb70bfb036d863e317e6e0d5843b983a6c8466 bom.cdx.json
|
||||
ff008ab332bbcc6ac413739eb66529c3fcb1ca2d2503f8263bf5e0645d930118 fragment-layer1.dsse.json
|
||||
7884ea6f3a46a0870d8fc74a5e770bac49a9729a83175dbcf42ca14769b22fa0 fragment-layer1.json
|
||||
8813b84f072196808e644e6a8c54a81348b566054149b26a0055d8e63e0ae6aa fragment-layer2.dsse.json
|
||||
cb9783249cf18e8d8a227d288864d821c190005897a14212f21742c0f404208f fragment-layer2.json
|
||||
Reference in New Issue
Block a user