Add tests for SBOM generation determinism across multiple formats
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
@@ -0,0 +1,267 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://stella-ops.org/schemas/determinism-manifest/v1.json",
|
||||
"title": "StellaOps Determinism Manifest",
|
||||
"description": "Manifest tracking artifact reproducibility with canonical bytes hash, version stamps, and toolchain information",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"schemaVersion",
|
||||
"artifact",
|
||||
"canonicalHash",
|
||||
"toolchain",
|
||||
"generatedAt"
|
||||
],
|
||||
"properties": {
|
||||
"schemaVersion": {
|
||||
"type": "string",
|
||||
"const": "1.0",
|
||||
"description": "Version of this manifest schema"
|
||||
},
|
||||
"artifact": {
|
||||
"type": "object",
|
||||
"description": "Artifact being tracked for determinism",
|
||||
"required": ["type", "name", "version"],
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"sbom",
|
||||
"vex",
|
||||
"csaf",
|
||||
"verdict",
|
||||
"evidence-bundle",
|
||||
"airgap-bundle",
|
||||
"advisory-normalized",
|
||||
"attestation",
|
||||
"other"
|
||||
],
|
||||
"description": "Type of artifact"
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Artifact identifier or name",
|
||||
"minLength": 1
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Artifact version or timestamp",
|
||||
"minLength": 1
|
||||
},
|
||||
"format": {
|
||||
"type": "string",
|
||||
"description": "Artifact format (e.g., 'SPDX 3.0.1', 'CycloneDX 1.6', 'OpenVEX')",
|
||||
"examples": ["SPDX 3.0.1", "CycloneDX 1.6", "OpenVEX", "CSAF 2.0"]
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Additional artifact-specific metadata",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"canonicalHash": {
|
||||
"type": "object",
|
||||
"description": "Hash of the canonical representation of the artifact",
|
||||
"required": ["algorithm", "value", "encoding"],
|
||||
"properties": {
|
||||
"algorithm": {
|
||||
"type": "string",
|
||||
"enum": ["SHA-256", "SHA-384", "SHA-512"],
|
||||
"description": "Hash algorithm used"
|
||||
},
|
||||
"value": {
|
||||
"type": "string",
|
||||
"description": "Hex-encoded hash value",
|
||||
"pattern": "^[0-9a-f]{64,128}$"
|
||||
},
|
||||
"encoding": {
|
||||
"type": "string",
|
||||
"enum": ["hex", "base64"],
|
||||
"description": "Encoding of the hash value"
|
||||
}
|
||||
}
|
||||
},
|
||||
"inputs": {
|
||||
"type": "object",
|
||||
"description": "Version stamps of all inputs used to generate the artifact",
|
||||
"properties": {
|
||||
"feedSnapshotHash": {
|
||||
"type": "string",
|
||||
"description": "SHA-256 hash of the vulnerability feed snapshot used",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
},
|
||||
"policyManifestHash": {
|
||||
"type": "string",
|
||||
"description": "SHA-256 hash of the policy manifest used",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
},
|
||||
"sourceCodeHash": {
|
||||
"type": "string",
|
||||
"description": "Git commit SHA or source code hash",
|
||||
"pattern": "^[0-9a-f]{40,64}$"
|
||||
},
|
||||
"dependencyLockfileHash": {
|
||||
"type": "string",
|
||||
"description": "Hash of dependency lockfile (e.g., package-lock.json, Cargo.lock)",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
},
|
||||
"baseImageDigest": {
|
||||
"type": "string",
|
||||
"description": "Container base image digest (sha256:...)",
|
||||
"pattern": "^sha256:[0-9a-f]{64}$"
|
||||
},
|
||||
"vexDocumentHashes": {
|
||||
"type": "array",
|
||||
"description": "Hashes of all VEX documents used as input",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
}
|
||||
},
|
||||
"custom": {
|
||||
"type": "object",
|
||||
"description": "Custom input hashes specific to artifact type",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"toolchain": {
|
||||
"type": "object",
|
||||
"description": "Toolchain version information",
|
||||
"required": ["platform", "components"],
|
||||
"properties": {
|
||||
"platform": {
|
||||
"type": "string",
|
||||
"description": "Runtime platform (e.g., '.NET 10.0', 'Node.js 20.0')",
|
||||
"examples": [".NET 10.0.0", "Node.js 20.11.0", "Python 3.12.1"]
|
||||
},
|
||||
"components": {
|
||||
"type": "array",
|
||||
"description": "Toolchain component versions",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["name", "version"],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Component name",
|
||||
"examples": ["StellaOps.Scanner", "StellaOps.Policy.Engine", "CycloneDX Generator"]
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Semantic version or git SHA",
|
||||
"examples": ["1.2.3", "2.0.0-beta.1", "abc123def"]
|
||||
},
|
||||
"hash": {
|
||||
"type": "string",
|
||||
"description": "Optional: SHA-256 hash of the component binary",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"compiler": {
|
||||
"type": "object",
|
||||
"description": "Compiler information if applicable",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Compiler name (e.g., 'Roslyn', 'rustc')"
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "Compiler version"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"generatedAt": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "UTC timestamp when artifact was generated (ISO 8601)",
|
||||
"examples": ["2025-12-23T17:45:00Z"]
|
||||
},
|
||||
"reproducibility": {
|
||||
"type": "object",
|
||||
"description": "Reproducibility metadata",
|
||||
"properties": {
|
||||
"deterministicSeed": {
|
||||
"type": "integer",
|
||||
"description": "Deterministic random seed if used",
|
||||
"minimum": 0
|
||||
},
|
||||
"clockFixed": {
|
||||
"type": "boolean",
|
||||
"description": "Whether system clock was fixed during generation"
|
||||
},
|
||||
"orderingGuarantee": {
|
||||
"type": "string",
|
||||
"enum": ["stable", "sorted", "insertion", "unspecified"],
|
||||
"description": "Ordering guarantee for collections in output"
|
||||
},
|
||||
"normalizationRules": {
|
||||
"type": "array",
|
||||
"description": "Normalization rules applied (e.g., 'UTF-8', 'LF line endings', 'no whitespace')",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"examples": [
|
||||
["UTF-8 encoding", "LF line endings", "sorted JSON keys", "no trailing whitespace"]
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"verification": {
|
||||
"type": "object",
|
||||
"description": "Verification instructions for reproducing the artifact",
|
||||
"properties": {
|
||||
"command": {
|
||||
"type": "string",
|
||||
"description": "Command to regenerate the artifact",
|
||||
"examples": ["dotnet run --project Scanner -- scan container alpine:3.18"]
|
||||
},
|
||||
"expectedHash": {
|
||||
"type": "string",
|
||||
"description": "Expected SHA-256 hash after reproduction",
|
||||
"pattern": "^[0-9a-f]{64}$"
|
||||
},
|
||||
"baseline": {
|
||||
"type": "string",
|
||||
"description": "Baseline manifest file path for regression testing",
|
||||
"examples": ["tests/baselines/sbom-alpine-3.18.determinism.json"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"signatures": {
|
||||
"type": "array",
|
||||
"description": "Optional cryptographic signatures of this manifest",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["algorithm", "keyId", "signature"],
|
||||
"properties": {
|
||||
"algorithm": {
|
||||
"type": "string",
|
||||
"description": "Signature algorithm (e.g., 'ES256', 'RS256')"
|
||||
},
|
||||
"keyId": {
|
||||
"type": "string",
|
||||
"description": "Key identifier used for signing"
|
||||
},
|
||||
"signature": {
|
||||
"type": "string",
|
||||
"description": "Base64-encoded signature"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "UTC timestamp when signature was created"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user