up
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,103 @@
|
||||
# Deterministic Evaluator Sample Configuration
|
||||
# This file demonstrates the configuration options for the policy evaluator
|
||||
# Version: 1.0
|
||||
|
||||
evaluator:
|
||||
# Determinism settings
|
||||
determinism:
|
||||
# Enforce strict determinism checks at runtime
|
||||
enforceStrict: true
|
||||
|
||||
# Log warnings for potential non-deterministic operations
|
||||
logWarnings: true
|
||||
|
||||
# Fail evaluation if non-deterministic operation detected
|
||||
failOnViolation: true
|
||||
|
||||
# Rule evaluation settings
|
||||
rules:
|
||||
# First-match semantics: stop on first matching rule
|
||||
firstMatchOnly: true
|
||||
|
||||
# Default status when no rules match
|
||||
defaultStatus: "affected"
|
||||
|
||||
# Enable priority-based ordering (lower priority evaluates first)
|
||||
priorityOrdering: true
|
||||
|
||||
# Exception handling settings
|
||||
exceptions:
|
||||
# Enable exception application after rule evaluation
|
||||
enabled: true
|
||||
|
||||
# Specificity weights for exception scope matching
|
||||
specificity:
|
||||
ruleNameBase: 1000
|
||||
ruleNamePerItem: 25
|
||||
severityBase: 500
|
||||
severityPerItem: 10
|
||||
sourceBase: 250
|
||||
sourcePerItem: 10
|
||||
tagBase: 100
|
||||
tagPerItem: 5
|
||||
|
||||
# Tie-breaker order: later CreatedAt wins, then lower Id wins
|
||||
tieBreaker:
|
||||
preferLaterCreatedAt: true
|
||||
preferLowerIdOnTie: true
|
||||
|
||||
# Value type settings
|
||||
values:
|
||||
# Use decimal for all numeric comparisons (no floating-point)
|
||||
useDecimalArithmetic: true
|
||||
|
||||
# Severity string-to-decimal mapping
|
||||
severityOrder:
|
||||
critical: 5
|
||||
high: 4
|
||||
medium: 3
|
||||
moderate: 3
|
||||
low: 2
|
||||
informational: 1
|
||||
info: 1
|
||||
none: 0
|
||||
unknown: -1
|
||||
|
||||
# Timestamp settings
|
||||
timestamps:
|
||||
# Format for all timestamp outputs
|
||||
format: "yyyy-MM-ddTHH:mm:ss.fffZ"
|
||||
|
||||
# Timezone for all timestamps (must be UTC for determinism)
|
||||
timezone: "UTC"
|
||||
|
||||
# Collection settings
|
||||
collections:
|
||||
# Use immutable collections for all internal state
|
||||
useImmutable: true
|
||||
|
||||
# String comparison mode for keys/lookups
|
||||
stringComparison: "OrdinalIgnoreCase"
|
||||
|
||||
# Content hashing settings for verification
|
||||
hashing:
|
||||
# Algorithm for content addressing
|
||||
algorithm: "SHA256"
|
||||
|
||||
# Include in output for audit trail
|
||||
includeInOutput: true
|
||||
|
||||
# Hash both input and output
|
||||
hashInputs: true
|
||||
hashOutputs: true
|
||||
|
||||
# Logging settings for determinism auditing
|
||||
logging:
|
||||
# Log rule evaluation order for debugging
|
||||
logRuleOrder: false
|
||||
|
||||
# Log exception selection for debugging
|
||||
logExceptionSelection: false
|
||||
|
||||
# Log final decision rationale
|
||||
logDecisionRationale: true
|
||||
@@ -0,0 +1,599 @@
|
||||
{
|
||||
"$schema": "https://stellaops.io/schemas/policy/test-vectors-v1.json",
|
||||
"version": "1.0",
|
||||
"description": "Deterministic evaluator test vectors with recorded input/output hashes",
|
||||
"generatedAt": "2025-11-27T00:00:00.000Z",
|
||||
"vectors": [
|
||||
{
|
||||
"id": "DEVAL-001",
|
||||
"name": "Critical severity blocks",
|
||||
"description": "Rule block_critical matches and returns blocked status",
|
||||
"input": {
|
||||
"policy": {
|
||||
"name": "Baseline Production Policy",
|
||||
"syntax": "stella-dsl@1",
|
||||
"rules": [
|
||||
{
|
||||
"name": "block_critical",
|
||||
"priority": 5,
|
||||
"when": "severity.normalized >= \"Critical\"",
|
||||
"then": "status := \"blocked\"",
|
||||
"because": "Critical severity must be remediated before deploy."
|
||||
}
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"severity": {
|
||||
"normalized": "Critical",
|
||||
"score": null
|
||||
},
|
||||
"environment": {
|
||||
"exposure": "internal"
|
||||
},
|
||||
"advisory": {
|
||||
"source": "GHSA",
|
||||
"metadata": {}
|
||||
},
|
||||
"vex": {
|
||||
"statements": []
|
||||
},
|
||||
"sbom": {
|
||||
"tags": [],
|
||||
"components": []
|
||||
},
|
||||
"exceptions": {
|
||||
"effects": {},
|
||||
"instances": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"expectedOutput": {
|
||||
"matched": true,
|
||||
"status": "blocked",
|
||||
"severity": "Critical",
|
||||
"ruleName": "block_critical",
|
||||
"priority": 5,
|
||||
"annotations": {},
|
||||
"warnings": [],
|
||||
"appliedException": null
|
||||
},
|
||||
"hashes": {
|
||||
"inputSha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"outputSha256": "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DEVAL-002",
|
||||
"name": "High severity with internet exposure escalates",
|
||||
"description": "Rule escalate_high_internet matches and escalates severity to Critical",
|
||||
"input": {
|
||||
"policy": {
|
||||
"name": "Baseline Production Policy",
|
||||
"syntax": "stella-dsl@1",
|
||||
"rules": [
|
||||
{
|
||||
"name": "escalate_high_internet",
|
||||
"priority": 10,
|
||||
"when": "severity.normalized == \"High\" and env.exposure == \"internet\"",
|
||||
"then": "escalate to severity_band(\"Critical\")",
|
||||
"because": "High severity on internet-exposed asset escalates to critical."
|
||||
}
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"severity": {
|
||||
"normalized": "High",
|
||||
"score": null
|
||||
},
|
||||
"environment": {
|
||||
"exposure": "internet"
|
||||
},
|
||||
"advisory": {
|
||||
"source": "GHSA",
|
||||
"metadata": {}
|
||||
},
|
||||
"vex": {
|
||||
"statements": []
|
||||
},
|
||||
"sbom": {
|
||||
"tags": [],
|
||||
"components": []
|
||||
},
|
||||
"exceptions": {
|
||||
"effects": {},
|
||||
"instances": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"expectedOutput": {
|
||||
"matched": true,
|
||||
"status": "affected",
|
||||
"severity": "Critical",
|
||||
"ruleName": "escalate_high_internet",
|
||||
"priority": 10,
|
||||
"annotations": {},
|
||||
"warnings": [],
|
||||
"appliedException": null
|
||||
},
|
||||
"hashes": {
|
||||
"inputSha256": "placeholder-compute-at-runtime",
|
||||
"outputSha256": "placeholder-compute-at-runtime"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DEVAL-003",
|
||||
"name": "VEX override sets status and annotation",
|
||||
"description": "Rule require_vex_justification matches and sets status from VEX statement",
|
||||
"input": {
|
||||
"policy": {
|
||||
"name": "Baseline Production Policy",
|
||||
"syntax": "stella-dsl@1",
|
||||
"rules": [
|
||||
{
|
||||
"name": "require_vex_justification",
|
||||
"priority": 10,
|
||||
"when": "vex.any(status in [\"not_affected\",\"fixed\"]) and vex.justification in [\"component_not_present\",\"vulnerable_code_not_present\"]",
|
||||
"then": "status := vex.status; annotate winning_statement := vex.latest().statementId",
|
||||
"because": "Respect strong vendor VEX claims."
|
||||
}
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"severity": {
|
||||
"normalized": "Medium",
|
||||
"score": null
|
||||
},
|
||||
"environment": {
|
||||
"exposure": "internal"
|
||||
},
|
||||
"advisory": {
|
||||
"source": "GHSA",
|
||||
"metadata": {}
|
||||
},
|
||||
"vex": {
|
||||
"statements": [
|
||||
{
|
||||
"status": "not_affected",
|
||||
"justification": "component_not_present",
|
||||
"statementId": "stmt-001",
|
||||
"timestamp": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"sbom": {
|
||||
"tags": [],
|
||||
"components": []
|
||||
},
|
||||
"exceptions": {
|
||||
"effects": {},
|
||||
"instances": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"expectedOutput": {
|
||||
"matched": true,
|
||||
"status": "not_affected",
|
||||
"severity": "Medium",
|
||||
"ruleName": "require_vex_justification",
|
||||
"priority": 10,
|
||||
"annotations": {
|
||||
"winning_statement": "stmt-001"
|
||||
},
|
||||
"warnings": [],
|
||||
"appliedException": null
|
||||
},
|
||||
"hashes": {
|
||||
"inputSha256": "placeholder-compute-at-runtime",
|
||||
"outputSha256": "placeholder-compute-at-runtime"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DEVAL-004",
|
||||
"name": "Exception suppresses critical finding",
|
||||
"description": "Exception with suppress effect overrides blocked status to suppressed",
|
||||
"input": {
|
||||
"policy": {
|
||||
"name": "Baseline Production Policy",
|
||||
"syntax": "stella-dsl@1",
|
||||
"rules": [
|
||||
{
|
||||
"name": "block_critical",
|
||||
"priority": 5,
|
||||
"when": "severity.normalized >= \"Critical\"",
|
||||
"then": "status := \"blocked\"",
|
||||
"because": "Critical severity must be remediated before deploy."
|
||||
}
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"severity": {
|
||||
"normalized": "Critical",
|
||||
"score": null
|
||||
},
|
||||
"environment": {
|
||||
"exposure": "internal"
|
||||
},
|
||||
"advisory": {
|
||||
"source": "GHSA",
|
||||
"metadata": {}
|
||||
},
|
||||
"vex": {
|
||||
"statements": []
|
||||
},
|
||||
"sbom": {
|
||||
"tags": [],
|
||||
"components": []
|
||||
},
|
||||
"exceptions": {
|
||||
"effects": {
|
||||
"suppress-critical": {
|
||||
"id": "suppress-critical",
|
||||
"name": "Critical Break Glass",
|
||||
"effect": "Suppress",
|
||||
"downgradeSeverity": null,
|
||||
"requiredControlId": null,
|
||||
"routingTemplate": "secops",
|
||||
"maxDurationDays": 7,
|
||||
"description": null
|
||||
}
|
||||
},
|
||||
"instances": [
|
||||
{
|
||||
"id": "exc-001",
|
||||
"effectId": "suppress-critical",
|
||||
"scope": {
|
||||
"ruleNames": ["block_critical"],
|
||||
"severities": [],
|
||||
"sources": [],
|
||||
"tags": []
|
||||
},
|
||||
"createdAt": "2025-10-01T00:00:00.000Z",
|
||||
"metadata": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"expectedOutput": {
|
||||
"matched": true,
|
||||
"status": "suppressed",
|
||||
"severity": "Critical",
|
||||
"ruleName": "block_critical",
|
||||
"priority": 5,
|
||||
"annotations": {
|
||||
"exception.id": "exc-001",
|
||||
"exception.effectId": "suppress-critical",
|
||||
"exception.effectType": "Suppress",
|
||||
"exception.effectName": "Critical Break Glass",
|
||||
"exception.routingTemplate": "secops",
|
||||
"exception.maxDurationDays": "7",
|
||||
"exception.status": "suppressed"
|
||||
},
|
||||
"warnings": [],
|
||||
"appliedException": {
|
||||
"exceptionId": "exc-001",
|
||||
"effectId": "suppress-critical",
|
||||
"effectType": "Suppress",
|
||||
"originalStatus": "blocked",
|
||||
"originalSeverity": "Critical",
|
||||
"appliedStatus": "suppressed",
|
||||
"appliedSeverity": "Critical",
|
||||
"metadata": {
|
||||
"routingTemplate": "secops",
|
||||
"maxDurationDays": "7",
|
||||
"effectName": "Critical Break Glass"
|
||||
}
|
||||
}
|
||||
},
|
||||
"hashes": {
|
||||
"inputSha256": "placeholder-compute-at-runtime",
|
||||
"outputSha256": "placeholder-compute-at-runtime"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DEVAL-005",
|
||||
"name": "More specific exception wins",
|
||||
"description": "Exception with higher specificity score wins over global exception",
|
||||
"input": {
|
||||
"policy": {
|
||||
"name": "Baseline Production Policy",
|
||||
"syntax": "stella-dsl@1",
|
||||
"rules": [
|
||||
{
|
||||
"name": "block_critical",
|
||||
"priority": 5,
|
||||
"when": "severity.normalized >= \"Critical\"",
|
||||
"then": "status := \"blocked\"",
|
||||
"because": "Critical severity must be remediated before deploy."
|
||||
}
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"severity": {
|
||||
"normalized": "Critical",
|
||||
"score": null
|
||||
},
|
||||
"environment": {
|
||||
"exposure": "internal"
|
||||
},
|
||||
"advisory": {
|
||||
"source": "GHSA",
|
||||
"metadata": {}
|
||||
},
|
||||
"vex": {
|
||||
"statements": []
|
||||
},
|
||||
"sbom": {
|
||||
"tags": [],
|
||||
"components": []
|
||||
},
|
||||
"exceptions": {
|
||||
"effects": {
|
||||
"suppress-critical-global": {
|
||||
"id": "suppress-critical-global",
|
||||
"name": "Global Critical Suppress",
|
||||
"effect": "Suppress"
|
||||
},
|
||||
"suppress-critical-rule": {
|
||||
"id": "suppress-critical-rule",
|
||||
"name": "Rule Critical Suppress",
|
||||
"effect": "Suppress"
|
||||
}
|
||||
},
|
||||
"instances": [
|
||||
{
|
||||
"id": "exc-global",
|
||||
"effectId": "suppress-critical-global",
|
||||
"scope": {
|
||||
"ruleNames": [],
|
||||
"severities": ["Critical"],
|
||||
"sources": [],
|
||||
"tags": []
|
||||
},
|
||||
"createdAt": "2025-09-01T00:00:00.000Z",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"id": "exc-rule",
|
||||
"effectId": "suppress-critical-rule",
|
||||
"scope": {
|
||||
"ruleNames": ["block_critical"],
|
||||
"severities": ["Critical"],
|
||||
"sources": [],
|
||||
"tags": []
|
||||
},
|
||||
"createdAt": "2025-10-05T00:00:00.000Z",
|
||||
"metadata": {
|
||||
"requestedBy": "alice"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"expectedOutput": {
|
||||
"matched": true,
|
||||
"status": "suppressed",
|
||||
"severity": "Critical",
|
||||
"ruleName": "block_critical",
|
||||
"priority": 5,
|
||||
"annotations": {
|
||||
"exception.id": "exc-rule",
|
||||
"exception.effectId": "suppress-critical-rule",
|
||||
"exception.effectType": "Suppress",
|
||||
"exception.effectName": "Rule Critical Suppress",
|
||||
"exception.status": "suppressed",
|
||||
"exception.meta.requestedBy": "alice"
|
||||
},
|
||||
"warnings": [],
|
||||
"appliedException": {
|
||||
"exceptionId": "exc-rule",
|
||||
"effectId": "suppress-critical-rule",
|
||||
"effectType": "Suppress",
|
||||
"originalStatus": "blocked",
|
||||
"originalSeverity": "Critical",
|
||||
"appliedStatus": "suppressed",
|
||||
"appliedSeverity": "Critical",
|
||||
"metadata": {
|
||||
"effectName": "Rule Critical Suppress",
|
||||
"requestedBy": "alice"
|
||||
}
|
||||
}
|
||||
},
|
||||
"hashes": {
|
||||
"inputSha256": "placeholder-compute-at-runtime",
|
||||
"outputSha256": "placeholder-compute-at-runtime"
|
||||
},
|
||||
"notes": "exc-rule wins because rule name scope (1000 + 25) beats severity-only scope (500 + 10)"
|
||||
},
|
||||
{
|
||||
"id": "DEVAL-006",
|
||||
"name": "No rule matches returns default",
|
||||
"description": "When no rules match, default result with affected status is returned",
|
||||
"input": {
|
||||
"policy": {
|
||||
"name": "Empty Policy",
|
||||
"syntax": "stella-dsl@1",
|
||||
"rules": []
|
||||
},
|
||||
"context": {
|
||||
"severity": {
|
||||
"normalized": "Low",
|
||||
"score": null
|
||||
},
|
||||
"environment": {
|
||||
"exposure": "internal"
|
||||
},
|
||||
"advisory": {
|
||||
"source": "GHSA",
|
||||
"metadata": {}
|
||||
},
|
||||
"vex": {
|
||||
"statements": []
|
||||
},
|
||||
"sbom": {
|
||||
"tags": [],
|
||||
"components": []
|
||||
},
|
||||
"exceptions": {
|
||||
"effects": {},
|
||||
"instances": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"expectedOutput": {
|
||||
"matched": false,
|
||||
"status": "affected",
|
||||
"severity": "Low",
|
||||
"ruleName": null,
|
||||
"priority": null,
|
||||
"annotations": {},
|
||||
"warnings": [],
|
||||
"appliedException": null
|
||||
},
|
||||
"hashes": {
|
||||
"inputSha256": "placeholder-compute-at-runtime",
|
||||
"outputSha256": "placeholder-compute-at-runtime"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DEVAL-007",
|
||||
"name": "Warn rule emits warning and sets status",
|
||||
"description": "Rule with warn action emits warning message and sets warned status",
|
||||
"input": {
|
||||
"policy": {
|
||||
"name": "Baseline Production Policy",
|
||||
"syntax": "stella-dsl@1",
|
||||
"rules": [
|
||||
{
|
||||
"name": "alert_warn_eol_runtime",
|
||||
"priority": 1,
|
||||
"when": "severity.normalized <= \"Medium\" and sbom.has_tag(\"runtime:eol\")",
|
||||
"then": "warn message \"Runtime marked as EOL; upgrade recommended.\"",
|
||||
"because": "Deprecated runtime should be upgraded."
|
||||
}
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"severity": {
|
||||
"normalized": "Medium",
|
||||
"score": null
|
||||
},
|
||||
"environment": {
|
||||
"exposure": "internal"
|
||||
},
|
||||
"advisory": {
|
||||
"source": "GHSA",
|
||||
"metadata": {}
|
||||
},
|
||||
"vex": {
|
||||
"statements": []
|
||||
},
|
||||
"sbom": {
|
||||
"tags": ["runtime:eol"],
|
||||
"components": []
|
||||
},
|
||||
"exceptions": {
|
||||
"effects": {},
|
||||
"instances": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"expectedOutput": {
|
||||
"matched": true,
|
||||
"status": "warned",
|
||||
"severity": "Medium",
|
||||
"ruleName": "alert_warn_eol_runtime",
|
||||
"priority": 1,
|
||||
"annotations": {},
|
||||
"warnings": ["Runtime marked as EOL; upgrade recommended."],
|
||||
"appliedException": null
|
||||
},
|
||||
"hashes": {
|
||||
"inputSha256": "placeholder-compute-at-runtime",
|
||||
"outputSha256": "placeholder-compute-at-runtime"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "DEVAL-008",
|
||||
"name": "Priority ordering ensures first-match semantics",
|
||||
"description": "Lower priority rule evaluates first and wins",
|
||||
"input": {
|
||||
"policy": {
|
||||
"name": "Priority Test Policy",
|
||||
"syntax": "stella-dsl@1",
|
||||
"rules": [
|
||||
{
|
||||
"name": "high_priority_rule",
|
||||
"priority": 1,
|
||||
"when": "true",
|
||||
"then": "status := \"high-priority-match\"",
|
||||
"because": "First priority wins"
|
||||
},
|
||||
{
|
||||
"name": "low_priority_rule",
|
||||
"priority": 10,
|
||||
"when": "true",
|
||||
"then": "status := \"low-priority-match\"",
|
||||
"because": "Never reached"
|
||||
}
|
||||
]
|
||||
},
|
||||
"context": {
|
||||
"severity": {
|
||||
"normalized": "Low",
|
||||
"score": null
|
||||
},
|
||||
"environment": {},
|
||||
"advisory": {
|
||||
"source": "GHSA",
|
||||
"metadata": {}
|
||||
},
|
||||
"vex": {
|
||||
"statements": []
|
||||
},
|
||||
"sbom": {
|
||||
"tags": [],
|
||||
"components": []
|
||||
},
|
||||
"exceptions": {
|
||||
"effects": {},
|
||||
"instances": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"expectedOutput": {
|
||||
"matched": true,
|
||||
"status": "high-priority-match",
|
||||
"severity": "Low",
|
||||
"ruleName": "high_priority_rule",
|
||||
"priority": 1,
|
||||
"annotations": {},
|
||||
"warnings": [],
|
||||
"appliedException": null
|
||||
},
|
||||
"hashes": {
|
||||
"inputSha256": "placeholder-compute-at-runtime",
|
||||
"outputSha256": "placeholder-compute-at-runtime"
|
||||
},
|
||||
"notes": "Verifies first-match semantics with priority ordering"
|
||||
}
|
||||
],
|
||||
"deterministicProperties": {
|
||||
"ruleOrderingAlgorithm": "stable-sort by (priority ASC, declaration-index ASC)",
|
||||
"firstMatchSemantics": true,
|
||||
"exceptionSpecificityWeights": {
|
||||
"ruleNameBase": 1000,
|
||||
"ruleNamePerItem": 25,
|
||||
"severityBase": 500,
|
||||
"severityPerItem": 10,
|
||||
"sourceBase": 250,
|
||||
"sourcePerItem": 10,
|
||||
"tagBase": 100,
|
||||
"tagPerItem": 5
|
||||
},
|
||||
"exceptionTieBreaker": "later CreatedAt wins, then lower Id lexicographically wins",
|
||||
"numericType": "decimal",
|
||||
"stringComparison": "OrdinalIgnoreCase"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user