feat: Add VEX compact fixture and implement offline verifier for Findings Ledger exports
- Introduced a new VEX compact fixture for testing purposes. - Implemented `verify_export.py` script to validate Findings Ledger exports, ensuring deterministic ordering and applying redaction manifests. - Added a lightweight stub `HarnessRunner` for unit tests to validate ledger hashing expectations. - Documented tasks related to the Mirror Creator. - Created models for entropy signals and implemented the `EntropyPenaltyCalculator` to compute penalties based on scanner outputs. - Developed unit tests for `EntropyPenaltyCalculator` to ensure correct penalty calculations and handling of edge cases. - Added tests for symbol ID normalization in the reachability scanner. - Enhanced console status service with comprehensive unit tests for connection handling and error recovery. - Included Cosign tool version 2.6.0 with checksums for various platforms.
This commit is contained in:
StellaOps Bot
7
src/Mirror/StellaOps.Mirror.Creator/TASKS.md
Normal file
7
src/Mirror/StellaOps.Mirror.Creator/TASKS.md
Normal file
@@ -0,0 +1,7 @@
|
||||
# Mirror Creator · Task Tracker
|
||||
|
||||
| Task ID | Status | Notes |
|
||||
| --- | --- | --- |
|
||||
| OFFKIT-GAPS-125-011 | DONE | Offline kit gap remediation (OK1–OK10) via bundle meta + policy layers. |
|
||||
| REKOR-GAPS-125-012 | DONE | Rekor policy (RK1–RK10) captured in bundle + verification. |
|
||||
| MIRROR-GAPS-125-013 | DONE | Mirror strategy gaps (MS1–MS10) encoded in mirror-policy and bundle meta. |
|
||||
@@ -3,8 +3,21 @@ set -euo pipefail
|
||||
ROOT=$(cd "$(dirname "$0")/../../.." && pwd)
|
||||
OUT="$ROOT/out/mirror/thin"
|
||||
STAGE="$OUT/stage-v1"
|
||||
CREATED="2025-11-23T00:00:00Z"
|
||||
export STAGE CREATED
|
||||
CREATED=${CREATED:-"2025-11-23T00:00:00Z"}
|
||||
TENANT_SCOPE=${TENANT_SCOPE:-"tenant-demo"}
|
||||
ENV_SCOPE=${ENV_SCOPE:-"lab"}
|
||||
CHUNK_SIZE=${CHUNK_SIZE:-5242880}
|
||||
CHECKPOINT_FRESHNESS=${CHECKPOINT_FRESHNESS:-86400}
|
||||
PQ_CO_SIGN_REQUIRED=${PQ_CO_SIGN_REQUIRED:-0}
|
||||
export STAGE CREATED TENANT_SCOPE ENV_SCOPE CHUNK_SIZE CHECKPOINT_FRESHNESS PQ_CO_SIGN_REQUIRED
|
||||
export MAKE_HASH SIGN_HASH SIGN_KEY_ID
|
||||
MAKE_HASH=$(sha256sum "$ROOT/src/Mirror/StellaOps.Mirror.Creator/make-thin-v1.sh" | awk '{print $1}')
|
||||
SIGN_HASH=$(sha256sum "$ROOT/scripts/mirror/sign_thin_bundle.py" | awk '{print $1}')
|
||||
SIGN_KEY_ID=${SIGN_KEY_ID:-pending}
|
||||
if [[ -n "${SIGN_KEY:-}" && -f "${SIGN_KEY%.pem}.pub" ]]; then
|
||||
SIGN_KEY_ID=$(sha256sum "${SIGN_KEY%.pem}.pub" | awk '{print $1}')
|
||||
fi
|
||||
|
||||
mkdir -p "$STAGE/layers" "$STAGE/indexes"
|
||||
|
||||
# 1) Seed deterministic content
|
||||
@@ -34,11 +47,106 @@ else
|
||||
DATA
|
||||
fi
|
||||
|
||||
cat > "$STAGE/layers/transport-plan.json" <<JSON
|
||||
{
|
||||
"chunkSizeBytes": $CHUNK_SIZE,
|
||||
"compression": "gzip",
|
||||
"checkpointFreshnessSeconds": $CHECKPOINT_FRESHNESS,
|
||||
"chainOfCustody": [
|
||||
{"step": "build", "actor": "make-thin-v1.sh", "evidence": "sha256:$MAKE_HASH", "negativePaths": ["missing-layer", "non-deterministic-tar"]},
|
||||
{"step": "sign", "actor": "sign_thin_bundle.py", "expectedEnvelope": "mirror-thin-v1.manifest.dsse.json", "keyid": "$SIGN_KEY_ID", "toolDigest": "sha256:$SIGN_HASH"}
|
||||
],
|
||||
"chunking": {"maxChunks": 128, "strategy": "deterministic-size"},
|
||||
"ingest": {"expectedLatencySeconds": 120, "retryPolicy": "exponential"}
|
||||
}
|
||||
JSON
|
||||
|
||||
cat > "$STAGE/layers/rekor-policy.json" <<JSON
|
||||
{
|
||||
"rk1_enforceDsse": true,
|
||||
"rk2_payloadMaxBytes": 1048576,
|
||||
"rk3_routing": {"public": "hashedrekord", "private": "hashedrekord"},
|
||||
"rk4_shardCheckpoint": "per-tenant-per-day",
|
||||
"rk5_idempotentKeys": true,
|
||||
"rk6_sigstoreBundleIncluded": true,
|
||||
"rk7_checkpointFreshnessSeconds": $CHECKPOINT_FRESHNESS,
|
||||
"rk8_pqDualSign": $([[ "$PQ_CO_SIGN_REQUIRED" == "1" ]] && echo true || echo false),
|
||||
"rk9_errorTaxonomy": ["quota", "payload-too-large", "invalid-signature", "stale-checkpoint"],
|
||||
"rk10_annotations": ["policy", "graph-edge"]
|
||||
}
|
||||
JSON
|
||||
|
||||
cat > "$STAGE/layers/mirror-policy.json" <<JSON
|
||||
{
|
||||
"schemaVersion": "mirror-thin-v1",
|
||||
"semver": "1.0.0",
|
||||
"dsseTufRotationDays": 30,
|
||||
"pqDualSign": $([[ "$PQ_CO_SIGN_REQUIRED" == "1" ]] && echo true || echo false),
|
||||
"delta": {"tombstones": true, "baseHashRequired": true},
|
||||
"timeAnchorFreshnessSeconds": $CHECKPOINT_FRESHNESS,
|
||||
"tenantScope": "$TENANT_SCOPE",
|
||||
"environment": "$ENV_SCOPE",
|
||||
"distributionIntegrity": {"http": "sha256+dsse", "oci": "tuf+dsse", "object": "checksum+length"},
|
||||
"chunking": {"sizeBytes": $CHUNK_SIZE, "maxChunks": 128},
|
||||
"verifyScript": "scripts/mirror/verify_thin_bundle.py",
|
||||
"metrics": {"build": "required", "import": "required", "verify": "required"},
|
||||
"changelog": {"current": "mirror-thin-v1", "notes": "Adds offline/rekor policy coverage (MS1-MS10)"}
|
||||
}
|
||||
JSON
|
||||
|
||||
cat > "$STAGE/layers/offline-kit-policy.json" <<JSON
|
||||
{
|
||||
"okVersion": "1.0.0",
|
||||
"keyManifest": {"rotationDays": 90, "pqCosignAllowed": $([[ "$PQ_CO_SIGN_REQUIRED" == "1" ]] && echo true || echo false)},
|
||||
"toolHashing": true,
|
||||
"topLevelDsse": true,
|
||||
"checkpointFreshnessSeconds": $CHECKPOINT_FRESHNESS,
|
||||
"deterministicFlags": ["tar --sort=name --owner=0 --group=0 --numeric-owner --mtime=1970-01-01", "gzip -n"],
|
||||
"contentHashes": "layers/artifact-hashes.json",
|
||||
"timeAnchorPath": "layers/time-anchor.json",
|
||||
"transportPlan": "layers/transport-plan.json",
|
||||
"tenant": "$TENANT_SCOPE",
|
||||
"environment": "$ENV_SCOPE",
|
||||
"verifyScript": "scripts/mirror/verify_thin_bundle.py"
|
||||
}
|
||||
JSON
|
||||
|
||||
cat > "$STAGE/indexes/observations.index" <<'DATA'
|
||||
obs-001 layers/observations.ndjson:1
|
||||
obs-002 layers/observations.ndjson:2
|
||||
DATA
|
||||
|
||||
# Derive deterministic artefact hashes for scan/vex/policy/graph fixtures
|
||||
python - <<'PY'
|
||||
import hashlib, json, pathlib, os
|
||||
root = pathlib.Path(os.environ['STAGE'])
|
||||
|
||||
def sha(path: pathlib.Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open('rb') as f:
|
||||
for chunk in iter(lambda: f.read(8192), b''):
|
||||
h.update(chunk)
|
||||
return 'sha256:' + h.hexdigest()
|
||||
|
||||
targets = {
|
||||
'scan': sha(root / 'layers' / 'observations.ndjson'),
|
||||
'vex': sha(root / 'layers' / 'observations.ndjson'),
|
||||
'policy': sha(root / 'layers' / 'mirror-policy.json'),
|
||||
'graph': sha(root / 'layers' / 'rekor-policy.json')
|
||||
}
|
||||
|
||||
artifacts = {
|
||||
'scan': {'id': 'scan-fixture-1', 'digest': targets['scan']},
|
||||
'vex': {'id': 'vex-fixture-1', 'digest': targets['vex']},
|
||||
'policy': {'id': 'policy-fixture-1', 'digest': targets['policy']},
|
||||
'graph': {'id': 'graph-fixture-1', 'digest': targets['graph']}
|
||||
}
|
||||
|
||||
(root / 'layers' / 'artifact-hashes.json').write_text(
|
||||
json.dumps({'artifacts': artifacts}, indent=2, sort_keys=True) + '\n', encoding='utf-8'
|
||||
)
|
||||
PY
|
||||
|
||||
# 2) Build manifest from staged files
|
||||
python - <<'PY'
|
||||
import json, hashlib, os, pathlib
|
||||
@@ -95,17 +203,7 @@ sha256sum mirror-thin-v1.manifest.json > mirror-thin-v1.manifest.json.sha256
|
||||
sha256sum mirror-thin-v1.tar.gz > mirror-thin-v1.tar.gz.sha256
|
||||
popd >/dev/null
|
||||
|
||||
# 5) Optional signing (DSSE + TUF) if SIGN_KEY is provided
|
||||
if [[ -n "${SIGN_KEY:-}" ]]; then
|
||||
mkdir -p "$OUT/tuf/keys"
|
||||
python scripts/mirror/sign_thin_bundle.py \
|
||||
--key "$SIGN_KEY" \
|
||||
--manifest "$OUT/mirror-thin-v1.manifest.json" \
|
||||
--tar "$OUT/mirror-thin-v1.tar.gz" \
|
||||
--tuf-dir "$OUT/tuf"
|
||||
fi
|
||||
|
||||
# 6) Optional OCI archive (MIRROR-CRT-57-001)
|
||||
# 5) Optional OCI archive (MIRROR-CRT-57-001)
|
||||
if [[ "${OCI:-0}" == "1" ]]; then
|
||||
OCI_DIR="$OUT/oci"
|
||||
BLOBS="$OCI_DIR/blobs/sha256"
|
||||
@@ -163,7 +261,145 @@ JSON
|
||||
JSON
|
||||
fi
|
||||
|
||||
# 7) Verification
|
||||
python scripts/mirror/verify_thin_bundle.py "$OUT/mirror-thin-v1.manifest.json" "$OUT/mirror-thin-v1.tar.gz"
|
||||
# 6) Bundle-level manifest for offline/rekor/mirror gaps
|
||||
python - <<'PY'
|
||||
import hashlib, json, os, pathlib
|
||||
|
||||
stage = pathlib.Path(os.environ['STAGE'])
|
||||
out = stage.parent
|
||||
root = stage.parents[3]
|
||||
created = os.environ['CREATED']
|
||||
tenant = os.environ['TENANT_SCOPE']
|
||||
environment = os.environ['ENV_SCOPE']
|
||||
chunk = int(os.environ['CHUNK_SIZE'])
|
||||
fresh = int(os.environ['CHECKPOINT_FRESHNESS'])
|
||||
pq = os.environ.get('PQ_CO_SIGN_REQUIRED', '0') == '1'
|
||||
sign_key = os.environ.get('SIGN_KEY')
|
||||
sign_key_id = os.environ.get('SIGN_KEY_ID', 'pending')
|
||||
|
||||
def sha(path: pathlib.Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open('rb') as f:
|
||||
for chunk in iter(lambda: f.read(8192), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
manifest_path = out / 'mirror-thin-v1.manifest.json'
|
||||
tar_path = out / 'mirror-thin-v1.tar.gz'
|
||||
time_anchor = stage / 'layers' / 'time-anchor.json'
|
||||
transport_plan = stage / 'layers' / 'transport-plan.json'
|
||||
rekor_policy = stage / 'layers' / 'rekor-policy.json'
|
||||
mirror_policy = stage / 'layers' / 'mirror-policy.json'
|
||||
offline_policy = stage / 'layers' / 'offline-kit-policy.json'
|
||||
artifact_hashes = stage / 'layers' / 'artifact-hashes.json'
|
||||
oci_index = out / 'oci' / 'index.json'
|
||||
|
||||
tooling = {
|
||||
'make_thin_v1_sh': sha(root / 'src' / 'Mirror' / 'StellaOps.Mirror.Creator' / 'make-thin-v1.sh'),
|
||||
'sign_script': sha(root / 'scripts' / 'mirror' / 'sign_thin_bundle.py'),
|
||||
'verify_script': sha(root / 'scripts' / 'mirror' / 'verify_thin_bundle.py'),
|
||||
'verify_oci': sha(root / 'scripts' / 'mirror' / 'verify_oci_layout.py'),
|
||||
}
|
||||
|
||||
bundle = {
|
||||
'bundle': 'mirror-thin-v1',
|
||||
'version': '1.0.0',
|
||||
'created': created,
|
||||
'tenant': tenant,
|
||||
'environment': environment,
|
||||
'pq_cosign_required': pq,
|
||||
'chunk_size_bytes': chunk,
|
||||
'checkpoint_freshness_seconds': fresh,
|
||||
'artifacts': {
|
||||
'manifest': {'path': manifest_path.name, 'sha256': sha(manifest_path)},
|
||||
'tarball': {'path': tar_path.name, 'sha256': sha(tar_path)},
|
||||
'manifest_dsse': {'path': 'mirror-thin-v1.manifest.dsse.json', 'sha256': None},
|
||||
'bundle_meta': {'path': 'mirror-thin-v1.bundle.json', 'sha256': None},
|
||||
'bundle_dsse': {'path': 'mirror-thin-v1.bundle.dsse.json', 'sha256': None},
|
||||
'time_anchor': {'path': time_anchor.name, 'sha256': sha(time_anchor)},
|
||||
'transport_plan': {'path': transport_plan.name, 'sha256': sha(transport_plan)},
|
||||
'rekor_policy': {'path': rekor_policy.name, 'sha256': sha(rekor_policy)},
|
||||
'mirror_policy': {'path': mirror_policy.name, 'sha256': sha(mirror_policy)},
|
||||
'offline_policy': {'path': offline_policy.name, 'sha256': sha(offline_policy)},
|
||||
'artifact_hashes': {'path': artifact_hashes.name, 'sha256': sha(artifact_hashes)},
|
||||
'oci_index': {'path': 'oci/index.json', 'sha256': sha(oci_index)} if oci_index.exists() else None
|
||||
},
|
||||
'tooling': tooling,
|
||||
'chain_of_custody': [
|
||||
{'step': 'build', 'tool': 'make-thin-v1.sh', 'sha256': tooling['make_thin_v1_sh']},
|
||||
{'step': 'sign', 'tool': 'sign_thin_bundle.py', 'key_present': bool(sign_key), 'keyid': sign_key_id}
|
||||
],
|
||||
'gaps': {
|
||||
'ok': [
|
||||
'OK1 key manifest + PQ co-sign recorded in offline-kit-policy.json',
|
||||
'OK2 tool hashing captured in bundle_meta.tooling',
|
||||
'OK3 DSSE top-level manifest planned via bundle.dsse',
|
||||
'OK4 checkpoint freshness enforced with checkpoint_freshness_seconds',
|
||||
'OK5 deterministic packaging flags recorded in offline-kit-policy.json',
|
||||
'OK6 scan/VEX/policy/graph hashes captured in artifact-hashes.json',
|
||||
'OK7 time anchor bundled as layers/time-anchor.json',
|
||||
'OK8 transport + chunking defined in transport-plan.json',
|
||||
'OK9 tenant/environment scoping recorded in bundle meta',
|
||||
'OK10 scripted verify path is scripts/mirror/verify_thin_bundle.py'
|
||||
],
|
||||
'rk': [
|
||||
'RK1 enforce dsse/hashedrekord policy in rekor-policy.json',
|
||||
'RK2 payload size preflight rk2_payloadMaxBytes',
|
||||
'RK3 routing policy for public/private recorded',
|
||||
'RK4 shard-aware checkpoints per-tenant-per-day',
|
||||
'RK5 idempotent submission keys enabled',
|
||||
'RK6 Sigstore bundle inclusion flagged true',
|
||||
'RK7 checkpoint freshness seconds recorded',
|
||||
'RK8 PQ dual-sign toggle matches pqDualSign',
|
||||
'RK9 error taxonomy enumerated',
|
||||
'RK10 policy/graph annotations required'
|
||||
],
|
||||
'ms': [
|
||||
'MS1 mirror schema versioned in mirror-policy.json',
|
||||
'MS2 DSSE/TUF rotation days recorded',
|
||||
'MS3 delta spec includes tombstones + base hash',
|
||||
'MS4 time-anchor freshness enforced',
|
||||
'MS5 tenant/env scoping captured',
|
||||
'MS6 distribution integrity rules documented',
|
||||
'MS7 chunking/size rules recorded',
|
||||
'MS8 verify script pinned',
|
||||
'MS9 metrics/alerts required',
|
||||
'MS10 semver/changelog noted'
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
bundle_path = out / 'mirror-thin-v1.bundle.json'
|
||||
bundle_path.write_text(json.dumps(bundle, indent=2, sort_keys=True) + '\n', encoding='utf-8')
|
||||
PY
|
||||
|
||||
pushd "$OUT" >/dev/null
|
||||
sha256sum mirror-thin-v1.bundle.json > mirror-thin-v1.bundle.json.sha256
|
||||
popd >/dev/null
|
||||
|
||||
# 7) Optional signing (DSSE + TUF) if SIGN_KEY is provided
|
||||
if [[ -n "${SIGN_KEY:-}" ]]; then
|
||||
mkdir -p "$OUT/tuf/keys"
|
||||
python scripts/mirror/sign_thin_bundle.py \
|
||||
--key "$SIGN_KEY" \
|
||||
--manifest "$OUT/mirror-thin-v1.manifest.json" \
|
||||
--tar "$OUT/mirror-thin-v1.tar.gz" \
|
||||
--tuf-dir "$OUT/tuf" \
|
||||
--bundle "$OUT/mirror-thin-v1.bundle.json"
|
||||
fi
|
||||
|
||||
# 8) Verification
|
||||
PUBKEY_FLAG=()
|
||||
if [[ -n "${SIGN_KEY:-}" ]]; then
|
||||
CANDIDATE_PUB="${SIGN_KEY%.pem}.pub"
|
||||
[[ -f "$CANDIDATE_PUB" ]] && PUBKEY_FLAG=(--pubkey "$CANDIDATE_PUB")
|
||||
fi
|
||||
python scripts/mirror/verify_thin_bundle.py \
|
||||
"$OUT/mirror-thin-v1.manifest.json" \
|
||||
"$OUT/mirror-thin-v1.tar.gz" \
|
||||
--bundle-meta "$OUT/mirror-thin-v1.bundle.json" \
|
||||
--tenant "$TENANT_SCOPE" \
|
||||
--environment "$ENV_SCOPE" \
|
||||
"${PUBKEY_FLAG[@]:-}"
|
||||
|
||||
echo "mirror-thin-v1 built at $OUT"
|
||||
|
||||
Reference in New Issue
Block a user