feat: Add VEX compact fixture and implement offline verifier for Findings Ledger exports

- Introduced a new VEX compact fixture for testing purposes.
- Implemented `verify_export.py` script to validate Findings Ledger exports, ensuring deterministic ordering and applying redaction manifests.
- Added a lightweight stub `HarnessRunner` for unit tests to validate ledger hashing expectations.
- Documented tasks related to the Mirror Creator.
- Created models for entropy signals and implemented the `EntropyPenaltyCalculator` to compute penalties based on scanner outputs.
- Developed unit tests for `EntropyPenaltyCalculator` to ensure correct penalty calculations and handling of edge cases.
- Added tests for symbol ID normalization in the reachability scanner.
- Enhanced console status service with comprehensive unit tests for connection handling and error recovery.
- Included Cosign tool version 2.6.0 with checksums for various platforms.

scripts/mirror/README.md

@@ -1,9 +1,9 @@
 # Mirror signing helpers
 
-- `make-thin-v1.sh`: builds thin bundle v1, computes checksums, optional DSSE+TUF signing when `SIGN_KEY` is set, and runs verifier.
-- `sign_thin_bundle.py`: signs manifest (DSSE) and root/targets/snapshot/timestamp JSON using an Ed25519 PEM key.
-- `verify_thin_bundle.py`: checks SHA256 sidecars, manifest schema, tar determinism, and manifest/index digests.
-- `ci-sign.sh`: CI wrapper. Set `MIRROR_SIGN_KEY_B64` (base64-encoded Ed25519 PEM) and run; it builds, signs, and verifies in one step.
+- `make-thin-v1.sh`: builds thin bundle v1, computes checksums, emits bundle meta (offline/rekor/mirror gaps), optional DSSE+TUF signing when `SIGN_KEY` is set, and runs verifier.
+- `sign_thin_bundle.py`: signs manifest (DSSE), bundle meta (DSSE), and root/targets/snapshot/timestamp JSON using an Ed25519 PEM key.
+- `verify_thin_bundle.py`: checks SHA256 sidecars, manifest schema, tar determinism, required layers, optional bundle meta and DSSE signatures; accepts `--bundle-meta`, `--pubkey`, `--tenant`, `--environment`.
+- `ci-sign.sh`: CI wrapper. Set `MIRROR_SIGN_KEY_B64` (base64-encoded Ed25519 PEM) and run; it builds, signs, and verifies in one step, emitting `milestone.json` with manifest/tar/bundle hashes.
 - `verify_oci_layout.py`: validates OCI layout/index/manifest and blob digests when `OCI=1` is used.
 
 Artifacts live under `out/mirror/thin/`.

scripts/mirror/ci-sign.sh

@@ -23,12 +23,23 @@ chmod 600 "$KEYFILE"
 openssl pkey -in "$KEYFILE" -pubout -out "$KEYDIR/ci-ed25519.pub" >/dev/null 2>&1
 STAGE=${STAGE:-$ROOT/out/mirror/thin/stage-v1}
 CREATED=${CREATED:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}
-SIGN_KEY="$KEYFILE" STAGE="$STAGE" CREATED="$CREATED" "$ROOT/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh"
+TENANT_SCOPE=${TENANT_SCOPE:-tenant-demo}
+ENV_SCOPE=${ENV_SCOPE:-lab}
+CHUNK_SIZE=${CHUNK_SIZE:-5242880}
+CHECKPOINT_FRESHNESS=${CHECKPOINT_FRESHNESS:-86400}
+OCI=${OCI:-1}
+SIGN_KEY="$KEYFILE" STAGE="$STAGE" CREATED="$CREATED" TENANT_SCOPE="$TENANT_SCOPE" ENV_SCOPE="$ENV_SCOPE" CHUNK_SIZE="$CHUNK_SIZE" CHECKPOINT_FRESHNESS="$CHECKPOINT_FRESHNESS" OCI="$OCI" "$ROOT/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh"
 
 # Emit milestone summary with hashes for downstream consumers
 MANIFEST_PATH="$ROOT/out/mirror/thin/mirror-thin-v1.manifest.json"
 TAR_PATH="$ROOT/out/mirror/thin/mirror-thin-v1.tar.gz"
 DSSE_PATH="$ROOT/out/mirror/thin/mirror-thin-v1.manifest.dsse.json"
+BUNDLE_PATH="$ROOT/out/mirror/thin/mirror-thin-v1.bundle.json"
+BUNDLE_DSSE_PATH="$ROOT/out/mirror/thin/mirror-thin-v1.bundle.dsse.json"
+TRANSPORT_PATH="$ROOT/out/mirror/thin/stage-v1/layers/transport-plan.json"
+REKOR_POLICY_PATH="$ROOT/out/mirror/thin/stage-v1/layers/rekor-policy.json"
+MIRROR_POLICY_PATH="$ROOT/out/mirror/thin/stage-v1/layers/mirror-policy.json"
+OFFLINE_POLICY_PATH="$ROOT/out/mirror/thin/stage-v1/layers/offline-kit-policy.json"
 SUMMARY_PATH="$ROOT/out/mirror/thin/milestone.json"
 
 sha256() {
@@ -41,7 +52,15 @@ cat > "$SUMMARY_PATH" <<JSON
   "manifest": {"path": "$(basename "$MANIFEST_PATH")", "sha256": "$(sha256 "$MANIFEST_PATH")"},
   "tarball": {"path": "$(basename "$TAR_PATH")", "sha256": "$(sha256 "$TAR_PATH")"},
   "dsse": $( [[ -f "$DSSE_PATH" ]] && echo "{\"path\": \"$(basename "$DSSE_PATH")\", \"sha256\": \"$(sha256 "$DSSE_PATH")\"}" || echo "null" ),
+  "bundle": $( [[ -f "$BUNDLE_PATH" ]] && echo "{\"path\": \"$(basename "$BUNDLE_PATH")\", \"sha256\": \"$(sha256 "$BUNDLE_PATH")\"}" || echo "null" ),
+  "bundle_dsse": $( [[ -f "$BUNDLE_DSSE_PATH" ]] && echo "{\"path\": \"$(basename "$BUNDLE_DSSE_PATH")\", \"sha256\": \"$(sha256 "$BUNDLE_DSSE_PATH")\"}" || echo "null" ),
   "time_anchor": $( [[ -n "${TIME_ANCHOR_FILE:-}" && -f "$TIME_ANCHOR_FILE" ]] && echo "{\"path\": \"$(basename "$TIME_ANCHOR_FILE")\", \"sha256\": \"$(sha256 "$TIME_ANCHOR_FILE")\"}" || echo "null" )
+  ,"policies": {
+    "transport": {"path": "$(basename "$TRANSPORT_PATH")", "sha256": "$(sha256 "$TRANSPORT_PATH")"},
+    "rekor": {"path": "$(basename "$REKOR_POLICY_PATH")", "sha256": "$(sha256 "$REKOR_POLICY_PATH")"},
+    "mirror": {"path": "$(basename "$MIRROR_POLICY_PATH")", "sha256": "$(sha256 "$MIRROR_POLICY_PATH")"},
+    "offline": {"path": "$(basename "$OFFLINE_POLICY_PATH")", "sha256": "$(sha256 "$OFFLINE_POLICY_PATH")"}
+  }
 }
 JSON
 

scripts/mirror/sign_thin_bundle.py

@@ -11,6 +11,7 @@ Usage:
 
 Writes:
   - mirror-thin-v1.manifest.dsse.json
+  - mirror-thin-v1.bundle.dsse.json (optional, when --bundle is provided)
   - updates signatures in root.json, targets.json, snapshot.json, timestamp.json
 """
 import argparse, base64, json, pathlib, hashlib
@@ -46,6 +47,7 @@ def main():
     ap.add_argument("--manifest", required=True, type=pathlib.Path)
     ap.add_argument("--tar", required=True, type=pathlib.Path)
     ap.add_argument("--tuf-dir", required=True, type=pathlib.Path)
+    ap.add_argument("--bundle", required=False, type=pathlib.Path)
     args = ap.parse_args()
 
     key = load_key(args.key)
@@ -62,11 +64,23 @@ def main():
     dsse_path = args.manifest.with_suffix(".dsse.json")
     write_json(dsse_path, dsse)
 
+    if args.bundle:
+        bundle_bytes = args.bundle.read_bytes()
+        bundle_sig = sign_bytes(key, bundle_bytes)
+        bundle_dsse = {
+            "payloadType": "application/vnd.stellaops.mirror.bundle+json",
+            "payload": b64url(bundle_bytes),
+            "signatures": [{"keyid": keyid, "sig": b64url(bundle_sig)}],
+        }
+        bundle_dsse_path = args.bundle.with_suffix(".dsse.json")
+        write_json(bundle_dsse_path, bundle_dsse)
+
     # update TUF metadata
     for name in ["root.json", "targets.json", "snapshot.json", "timestamp.json"]:
         sign_tuf(args.tuf_dir / name, keyid, key)
 
-    print(f"Signed DSSE + TUF using keyid {keyid}; DSSE -> {dsse_path}")
+    extra = f", bundle DSSE -> {bundle_dsse_path}" if args.bundle else ""
+    print(f"Signed DSSE + TUF using keyid {keyid}; DSSE -> {dsse_path}{extra}")
 
 if __name__ == "__main__":
     main()

scripts/mirror/verify_thin_bundle.py

@@ -1,20 +1,59 @@
 #!/usr/bin/env python3
 """
-Simple verifier for mirror-thin-v1 artefacts.
+Verifier for mirror-thin-v1 artefacts and bundle meta.
+
 Checks:
-1) SHA256 of manifest and tarball matches provided .sha256 files.
-2) Manifest schema has required fields.
-3) Tarball contains manifest.json, layers/, indexes/ with deterministic tar headers (mtime=0, uid/gid=0, sorted paths).
-4) Tar content digests match manifest entries.
+1) SHA256 of manifest/tarball (and optional bundle meta) matches sidecars.
+2) Manifest schema contains required fields and required layer files exist.
+3) Tarball headers deterministic (sorted paths, uid/gid=0, mtime=0).
+4) Tar contents match manifest digests.
+5) Optional: verify DSSE signatures for manifest/bundle when a public key is provided.
+6) Optional: validate bundle meta (tenant/env scope, policy hashes, gap coverage counts).
 
 Usage:
-  python scripts/mirror/verify_thin_bundle.py out/mirror/thin/mirror-thin-v1.manifest.json out/mirror/thin/mirror-thin-v1.tar.gz
+  python scripts/mirror/verify_thin_bundle.py \
+    out/mirror/thin/mirror-thin-v1.manifest.json \
+    out/mirror/thin/mirror-thin-v1.tar.gz \
+    --bundle-meta out/mirror/thin/mirror-thin-v1.bundle.json \
+    --pubkey out/mirror/thin/tuf/keys/ci-ed25519.pub \
+    --tenant tenant-demo --environment lab
 
 Exit code 0 on success; non-zero on any check failure.
 """
-import json, tarfile, hashlib, sys, pathlib
+import argparse
+import base64
+import hashlib
+import json
+import pathlib
+import sys
+import tarfile
+from typing import Optional
+
+try:
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
+
+    CRYPTO_AVAILABLE = True
+except ImportError:  # pragma: no cover - surfaced as runtime guidance
+    CRYPTO_AVAILABLE = False
 
 REQUIRED_FIELDS = ["version", "created", "layers", "indexes"]
+REQUIRED_LAYER_FILES = {
+    "layers/observations.ndjson",
+    "layers/time-anchor.json",
+    "layers/transport-plan.json",
+    "layers/rekor-policy.json",
+    "layers/mirror-policy.json",
+    "layers/offline-kit-policy.json",
+    "layers/artifact-hashes.json",
+    "indexes/observations.index",
+}
+
+
+def _b64url_decode(data: str) -> bytes:
+    padding = "=" * (-len(data) % 4)
+    return base64.urlsafe_b64decode(data + padding)
+
 
 def sha256_file(path: pathlib.Path) -> str:
     h = hashlib.sha256()
@@ -23,20 +62,24 @@ def sha256_file(path: pathlib.Path) -> str:
             h.update(chunk)
     return h.hexdigest()
 
+
 def load_sha256_sidecar(path: pathlib.Path) -> str:
     sidecar = path.with_suffix(path.suffix + ".sha256")
     if not sidecar.exists():
         raise SystemExit(f"missing sidecar {sidecar}")
     return sidecar.read_text().strip().split()[0]
 
+
 def check_schema(manifest: dict):
     missing = [f for f in REQUIRED_FIELDS if f not in manifest]
     if missing:
         raise SystemExit(f"manifest missing fields: {missing}")
 
+
 def normalize(name: str) -> str:
     return name[2:] if name.startswith("./") else name
 
+
 def check_tar_determinism(tar_path: pathlib.Path):
     with tarfile.open(tar_path, "r:gz") as tf:
         names = [normalize(n) for n in tf.getnames()]
@@ -48,13 +91,21 @@ def check_tar_determinism(tar_path: pathlib.Path):
             if m.mtime != 0:
                 raise SystemExit(f"tar header mtime not zero for {m.name}")
 
+
+def check_required_layers(tar_path: pathlib.Path):
+    with tarfile.open(tar_path, "r:gz") as tf:
+        names = {normalize(n) for n in tf.getnames()}
+        for required in REQUIRED_LAYER_FILES:
+            if required not in names:
+                raise SystemExit(f"required file missing from bundle: {required}")
+
+
 def check_content_hashes(manifest: dict, tar_path: pathlib.Path):
     with tarfile.open(tar_path, "r:gz") as tf:
         def get(name: str):
             try:
                 return tf.getmember(name)
             except KeyError:
-                # retry with leading ./
                 return tf.getmember(f"./{name}")
         for layer in manifest.get("layers", []):
             name = layer["path"]
@@ -74,12 +125,96 @@ def check_content_hashes(manifest: dict, tar_path: pathlib.Path):
                 raise SystemExit(f"index digest mismatch {name}: {digest}")
 
 
+def load_pubkey(path: pathlib.Path) -> Ed25519PublicKey:
+    if not CRYPTO_AVAILABLE:
+        raise SystemExit("cryptography is required for DSSE verification; install before using --pubkey")
+    return serialization.load_pem_public_key(path.read_bytes())
+
+
+def verify_dsse(dsse_path: pathlib.Path, pubkey_path: pathlib.Path, expected_payload: pathlib.Path, expected_type: str):
+    dsse_obj = json.loads(dsse_path.read_text())
+    if dsse_obj.get("payloadType") != expected_type:
+        raise SystemExit(f"DSSE payloadType mismatch for {dsse_path}")
+    payload = _b64url_decode(dsse_obj.get("payload", ""))
+    if payload != expected_payload.read_bytes():
+        raise SystemExit(f"DSSE payload mismatch for {dsse_path}")
+    sigs = dsse_obj.get("signatures") or []
+    if not sigs:
+        raise SystemExit(f"DSSE missing signatures: {dsse_path}")
+    pub = load_pubkey(pubkey_path)
+    try:
+        pub.verify(_b64url_decode(sigs[0]["sig"]), payload)
+    except Exception as exc:  # pragma: no cover - cryptography raises InvalidSignature
+        raise SystemExit(f"DSSE signature verification failed for {dsse_path}: {exc}")
+
+
+def check_bundle_meta(meta_path: pathlib.Path, manifest_path: pathlib.Path, tar_path: pathlib.Path, tenant: Optional[str], environment: Optional[str]):
+    meta = json.loads(meta_path.read_text())
+    for field in ["bundle", "version", "artifacts", "gaps", "tooling"]:
+        if field not in meta:
+            raise SystemExit(f"bundle meta missing field {field}")
+    if tenant and meta.get("tenant") != tenant:
+        raise SystemExit(f"bundle tenant mismatch: {meta.get('tenant')} != {tenant}")
+    if environment and meta.get("environment") != environment:
+        raise SystemExit(f"bundle environment mismatch: {meta.get('environment')} != {environment}")
+
+    artifacts = meta["artifacts"]
+
+    def expect(name: str, path: pathlib.Path):
+        recorded = artifacts.get(name)
+        if not recorded:
+            raise SystemExit(f"bundle meta missing artifact entry: {name}")
+        expected = recorded.get("sha256")
+        if expected and expected != sha256_file(path):
+            raise SystemExit(f"bundle meta digest mismatch for {name}")
+
+    expect("manifest", manifest_path)
+    expect("tarball", tar_path)
+    for extra in ["time_anchor", "transport_plan", "rekor_policy", "mirror_policy", "offline_policy", "artifact_hashes"]:
+        rec = artifacts.get(extra)
+        if not rec:
+            raise SystemExit(f"bundle meta missing artifact entry: {extra}")
+        if not rec.get("path"):
+            raise SystemExit(f"bundle meta missing path for {extra}")
+
+    for group, expected_count in [("ok", 10), ("rk", 10), ("ms", 10)]:
+        if len(meta.get("gaps", {}).get(group, [])) != expected_count:
+            raise SystemExit(f"bundle meta gaps.{group} expected {expected_count} entries")
+
+    root_guess = manifest_path.parents[3] if len(manifest_path.parents) > 3 else manifest_path.parents[-1]
+    tool_expectations = {
+        'make_thin_v1_sh': root_guess / 'src' / 'Mirror' / 'StellaOps.Mirror.Creator' / 'make-thin-v1.sh',
+        'sign_script': root_guess / 'scripts' / 'mirror' / 'sign_thin_bundle.py',
+        'verify_script': root_guess / 'scripts' / 'mirror' / 'verify_thin_bundle.py',
+        'verify_oci': root_guess / 'scripts' / 'mirror' / 'verify_oci_layout.py'
+    }
+    for key, path in tool_expectations.items():
+        recorded = meta['tooling'].get(key)
+        if not recorded:
+            raise SystemExit(f"tool hash missing for {key}")
+        actual = sha256_file(path)
+        if recorded != actual:
+            raise SystemExit(f"tool hash mismatch for {key}")
+
+    if meta.get("checkpoint_freshness_seconds", 0) <= 0:
+        raise SystemExit("checkpoint_freshness_seconds must be positive")
+
+
 def main():
-    if len(sys.argv) != 3:
-        print(__doc__)
-        sys.exit(2)
-    manifest_path = pathlib.Path(sys.argv[1])
-    tar_path = pathlib.Path(sys.argv[2])
+    parser = argparse.ArgumentParser()
+    parser.add_argument("manifest", type=pathlib.Path)
+    parser.add_argument("tar", type=pathlib.Path)
+    parser.add_argument("--bundle-meta", type=pathlib.Path)
+    parser.add_argument("--pubkey", type=pathlib.Path)
+    parser.add_argument("--tenant", type=str)
+    parser.add_argument("--environment", type=str)
+    args = parser.parse_args()
+
+    manifest_path = args.manifest
+    tar_path = args.tar
+    bundle_meta = args.bundle_meta
+    bundle_dsse = bundle_meta.with_suffix(".dsse.json") if bundle_meta else None
+    manifest_dsse = manifest_path.with_suffix(".dsse.json")
 
     man_expected = load_sha256_sidecar(manifest_path)
     tar_expected = load_sha256_sidecar(tar_path)
@@ -91,8 +226,26 @@ def main():
     manifest = json.loads(manifest_path.read_text())
     check_schema(manifest)
     check_tar_determinism(tar_path)
+    check_required_layers(tar_path)
     check_content_hashes(manifest, tar_path)
+
+    if bundle_meta:
+        if not bundle_meta.exists():
+            raise SystemExit(f"bundle meta missing: {bundle_meta}")
+        meta_expected = load_sha256_sidecar(bundle_meta)
+        if sha256_file(bundle_meta) != meta_expected:
+            raise SystemExit("bundle meta sha256 mismatch")
+        check_bundle_meta(bundle_meta, manifest_path, tar_path, args.tenant, args.environment)
+
+    if args.pubkey:
+        pubkey = args.pubkey
+        if manifest_dsse.exists():
+            verify_dsse(manifest_dsse, pubkey, manifest_path, "application/vnd.stellaops.mirror.manifest+json")
+        if bundle_dsse and bundle_dsse.exists():
+            verify_dsse(bundle_dsse, pubkey, bundle_meta, "application/vnd.stellaops.mirror.bundle+json")
+
     print("OK: mirror-thin bundle verified")
 
+
 if __name__ == "__main__":
     main()