Files
git.stella-ops.org/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/PolicyEvaluatorTests.cs
StellaOps Bot d040c001ac
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
up
2025-11-28 19:23:54 +02:00

665 lines
25 KiB
C#

using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.Policy;
using StellaOps.PolicyDsl;
using StellaOps.Policy.Engine.Evaluation;
using StellaOps.Policy.Engine.Services;
using Xunit;
using Xunit.Sdk;
namespace StellaOps.Policy.Engine.Tests;
public sealed class PolicyEvaluatorTests
{
private static readonly string BaselinePolicy = """
policy "Baseline Production Policy" syntax "stella-dsl@1" {
metadata {
description = "Block critical, escalate high, enforce VEX justifications."
tags = ["baseline","production"]
}
profile severity {
map vendor_weight {
source "GHSA" => +0.5
source "OSV" => +0.0
}
env exposure_adjustments {
if env.exposure == "internet" then +0.5
}
}
rule block_critical priority 5 {
when severity.normalized >= "Critical"
then status := "blocked"
because "Critical severity must be remediated before deploy."
}
rule escalate_high_internet {
when severity.normalized == "High"
and env.exposure == "internet"
then escalate to severity_band("Critical")
because "High severity on internet-exposed asset escalates to critical."
}
rule require_vex_justification {
when vex.any(status in ["not_affected","fixed"])
and vex.justification in ["component_not_present","vulnerable_code_not_present"]
then status := vex.status
annotate winning_statement := vex.latest().statementId
because "Respect strong vendor VEX claims."
}
rule alert_warn_eol_runtime priority 1 {
when severity.normalized <= "Medium"
and sbom.has_tag("runtime:eol")
then warn message "Runtime marked as EOL; upgrade recommended."
because "Deprecated runtime should be upgraded."
}
rule block_ruby_dev priority 4 {
when sbom.any_component(ruby.group("development") and ruby.declared_only())
then status := "blocked"
because "Development-only Ruby gems without install evidence cannot ship."
}
rule warn_ruby_git_sources {
when sbom.any_component(ruby.source("git"))
then warn message "Git-sourced Ruby gem present; review required."
because "Git-sourced Ruby dependencies require explicit review."
}
}
""";
private readonly PolicyCompiler compiler = new();
private readonly PolicyEvaluationService evaluationService = new();
[Fact]
public void Evaluate_BlockCriticalRuleMatches()
{
var document = CompileBaseline();
var context = CreateContext(severity: "Critical", exposure: "internal");
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("block_critical", result.RuleName);
Assert.Equal("blocked", result.Status);
}
[Fact]
public void Evaluate_EscalateAdjustsSeverity()
{
var document = CompileBaseline();
var context = CreateContext(severity: "High", exposure: "internet");
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("escalate_high_internet", result.RuleName);
Assert.Equal("affected", result.Status);
Assert.Equal("Critical", result.Severity);
}
[Fact]
public void Evaluate_VexOverrideSetsStatusAndAnnotation()
{
var document = CompileBaseline();
var statements = ImmutableArray.Create(
new PolicyEvaluationVexStatement("not_affected", "component_not_present", "stmt-001"));
var context = CreateContext("Medium", "internal") with
{
Vex = new PolicyEvaluationVexEvidence(statements)
};
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("require_vex_justification", result.RuleName);
Assert.Equal("not_affected", result.Status);
Assert.Equal("stmt-001", result.Annotations["winning_statement"]);
}
[Fact]
public void Evaluate_WarnRuleEmitsWarning()
{
var document = CompileBaseline();
var tags = ImmutableHashSet.Create("runtime:eol");
var context = CreateContext("Medium", "internal") with
{
Sbom = new PolicyEvaluationSbom(tags)
};
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("alert_warn_eol_runtime", result.RuleName);
Assert.Equal("warned", result.Status);
Assert.Contains(result.Warnings, message => message.Contains("EOL", StringComparison.OrdinalIgnoreCase));
}
[Fact]
public void Evaluate_ExceptionSuppressesCriticalFinding()
{
var document = CompileBaseline();
var effect = new PolicyExceptionEffect(
Id: "suppress-critical",
Name: "Critical Break Glass",
Effect: PolicyExceptionEffectType.Suppress,
DowngradeSeverity: null,
RequiredControlId: null,
RoutingTemplate: "secops",
MaxDurationDays: 7,
Description: null);
var scope = PolicyEvaluationExceptionScope.Create(ruleNames: new[] { "block_critical" });
var instance = new PolicyEvaluationExceptionInstance(
Id: "exc-001",
EffectId: effect.Id,
Scope: scope,
CreatedAt: new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero),
Metadata: ImmutableDictionary<string, string>.Empty);
var exceptions = new PolicyEvaluationExceptions(
ImmutableDictionary<string, PolicyExceptionEffect>.Empty.Add(effect.Id, effect),
ImmutableArray.Create(instance));
var context = CreateContext("Critical", "internal", exceptions);
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("block_critical", result.RuleName);
Assert.Equal("suppressed", result.Status);
Assert.NotNull(result.AppliedException);
Assert.Equal("exc-001", result.AppliedException!.ExceptionId);
Assert.Equal("suppress-critical", result.AppliedException!.EffectId);
Assert.Equal("blocked", result.AppliedException!.OriginalStatus);
Assert.Equal("suppressed", result.AppliedException!.AppliedStatus);
Assert.Equal("suppressed", result.Annotations["exception.status"]);
}
[Fact]
public void Evaluate_ExceptionDowngradesSeverity()
{
var document = CompileBaseline();
var effect = new PolicyExceptionEffect(
Id: "downgrade-internet",
Name: "Downgrade High Internet",
Effect: PolicyExceptionEffectType.Downgrade,
DowngradeSeverity: PolicySeverity.Medium,
RequiredControlId: null,
RoutingTemplate: null,
MaxDurationDays: null,
Description: null);
var scope = PolicyEvaluationExceptionScope.Create(
ruleNames: new[] { "escalate_high_internet" },
severities: new[] { "High" },
sources: new[] { "GHSA" });
var instance = new PolicyEvaluationExceptionInstance(
Id: "exc-200",
EffectId: effect.Id,
Scope: scope,
CreatedAt: new DateTimeOffset(2025, 10, 2, 0, 0, 0, TimeSpan.Zero),
Metadata: ImmutableDictionary<string, string>.Empty);
var exceptions = new PolicyEvaluationExceptions(
ImmutableDictionary<string, PolicyExceptionEffect>.Empty.Add(effect.Id, effect),
ImmutableArray.Create(instance));
var context = CreateContext("High", "internet", exceptions);
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("escalate_high_internet", result.RuleName);
Assert.Equal("affected", result.Status);
Assert.Equal("Medium", result.Severity);
Assert.NotNull(result.AppliedException);
Assert.Equal("Critical", result.AppliedException!.OriginalSeverity);
Assert.Equal("Medium", result.AppliedException!.AppliedSeverity);
Assert.Equal("Medium", result.Annotations["exception.severity"]);
}
[Fact]
public void Evaluate_MoreSpecificExceptionWins()
{
var document = CompileBaseline();
var suppressGlobal = new PolicyExceptionEffect(
Id: "suppress-critical-global",
Name: "Global Critical Suppress",
Effect: PolicyExceptionEffectType.Suppress,
DowngradeSeverity: null,
RequiredControlId: null,
RoutingTemplate: null,
MaxDurationDays: null,
Description: null);
var suppressRule = new PolicyExceptionEffect(
Id: "suppress-critical-rule",
Name: "Rule Critical Suppress",
Effect: PolicyExceptionEffectType.Suppress,
DowngradeSeverity: null,
RequiredControlId: null,
RoutingTemplate: null,
MaxDurationDays: null,
Description: null);
var globalInstance = new PolicyEvaluationExceptionInstance(
Id: "exc-global",
EffectId: suppressGlobal.Id,
Scope: PolicyEvaluationExceptionScope.Create(severities: new[] { "Critical" }),
CreatedAt: new DateTimeOffset(2025, 9, 1, 0, 0, 0, TimeSpan.Zero),
Metadata: ImmutableDictionary<string, string>.Empty);
var ruleInstance = new PolicyEvaluationExceptionInstance(
Id: "exc-rule",
EffectId: suppressRule.Id,
Scope: PolicyEvaluationExceptionScope.Create(
ruleNames: new[] { "block_critical" },
severities: new[] { "Critical" }),
CreatedAt: new DateTimeOffset(2025, 10, 5, 0, 0, 0, TimeSpan.Zero),
Metadata: ImmutableDictionary<string, string>.Empty.Add("requestedBy", "alice"));
var effects = ImmutableDictionary<string, PolicyExceptionEffect>.Empty
.Add(suppressGlobal.Id, suppressGlobal)
.Add(suppressRule.Id, suppressRule);
var exceptions = new PolicyEvaluationExceptions(
effects,
ImmutableArray.Create(globalInstance, ruleInstance));
var context = CreateContext("Critical", "internal", exceptions);
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("suppressed", result.Status);
Assert.NotNull(result.AppliedException);
Assert.Equal("exc-rule", result.AppliedException!.ExceptionId);
Assert.Equal("Rule Critical Suppress", result.AppliedException!.Metadata["effectName"]);
Assert.Equal("alice", result.AppliedException!.Metadata["requestedBy"]);
Assert.Equal("alice", result.Annotations["exception.meta.requestedBy"]);
}
[Fact]
public void Evaluate_RubyDevComponentBlocked()
{
var document = CompileBaseline();
var component = CreateRubyComponent(
name: "dev-only",
version: "1.0.0",
groups: "development;test",
declaredOnly: true,
source: "https://rubygems.org/",
capabilities: new[] { "exec" });
var context = CreateContext("Medium", "internal") with
{
Sbom = new PolicyEvaluationSbom(
ImmutableHashSet<string>.Empty.WithComparer(StringComparer.OrdinalIgnoreCase),
ImmutableArray.Create(component))
};
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("block_ruby_dev", result.RuleName);
Assert.Equal("blocked", result.Status);
}
[Fact]
public void Evaluate_RubyGitComponentWarns()
{
var document = CompileBaseline();
var component = CreateRubyComponent(
name: "git-gem",
version: "0.5.0",
groups: "default",
declaredOnly: false,
source: "git:https://github.com/example/git-gem.git@0123456789abcdef0123456789abcdef01234567",
capabilities: Array.Empty<string>(),
schedulerCapabilities: new[] { "sidekiq" });
var context = CreateContext("Low", "internal") with
{
Sbom = new PolicyEvaluationSbom(
ImmutableHashSet<string>.Empty.WithComparer(StringComparer.OrdinalIgnoreCase),
ImmutableArray.Create(component))
};
var result = evaluationService.Evaluate(document, context);
Assert.True(result.Matched);
Assert.Equal("warn_ruby_git_sources", result.RuleName);
Assert.Equal("warned", result.Status);
Assert.Contains(result.Warnings, warning => warning.Contains("Git-sourced", StringComparison.OrdinalIgnoreCase));
}
private PolicyIrDocument CompileBaseline()
{
var compilation = compiler.Compile(BaselinePolicy);
if (!compilation.Success)
{
Console.WriteLine(Describe(compilation.Diagnostics));
}
Assert.True(compilation.Success, Describe(compilation.Diagnostics));
return Assert.IsType<PolicyIrDocument>(compilation.Document);
}
private static PolicyEvaluationContext CreateContext(string severity, string exposure, PolicyEvaluationExceptions? exceptions = null)
{
return new PolicyEvaluationContext(
new PolicyEvaluationSeverity(severity),
new PolicyEvaluationEnvironment(new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
["exposure"] = exposure
}.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase)),
new PolicyEvaluationAdvisory("GHSA", ImmutableDictionary<string, string>.Empty),
PolicyEvaluationVexEvidence.Empty,
PolicyEvaluationSbom.Empty,
exceptions ?? PolicyEvaluationExceptions.Empty);
}
private static string Describe(ImmutableArray<PolicyIssue> issues) =>
string.Join(" | ", issues.Select(issue => $"{issue.Severity}:{issue.Code}:{issue.Message}"));
private static PolicyEvaluationComponent CreateRubyComponent(
string name,
string version,
string groups,
bool declaredOnly,
string source,
IEnumerable<string>? capabilities = null,
IEnumerable<string>? schedulerCapabilities = null)
{
var metadataBuilder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(groups))
{
metadataBuilder["groups"] = groups;
}
metadataBuilder["declaredOnly"] = declaredOnly ? "true" : "false";
if (!string.IsNullOrWhiteSpace(source))
{
metadataBuilder["source"] = source.Trim();
}
if (capabilities is not null)
{
foreach (var capability in capabilities)
{
if (!string.IsNullOrWhiteSpace(capability))
{
metadataBuilder[$"capability.{capability.Trim()}"] = "true";
}
}
}
if (schedulerCapabilities is not null)
{
var schedulerList = string.Join(
';',
schedulerCapabilities
.Where(static s => !string.IsNullOrWhiteSpace(s))
.Select(static s => s.Trim()));
if (!string.IsNullOrWhiteSpace(schedulerList))
{
metadataBuilder["capability.scheduler"] = schedulerList;
}
}
metadataBuilder["lockfile"] = "Gemfile.lock";
return new PolicyEvaluationComponent(
name,
version,
"gem",
$"pkg:gem/{name}@{version}",
metadataBuilder.ToImmutable());
}
private static PolicyEvaluationComponent CreateMacOsComponent(
string bundleId,
string version,
bool sandboxed = false,
bool hardenedRuntime = false,
string? teamId = null,
string? codeResourcesHash = null,
IEnumerable<string>? categories = null,
IEnumerable<string>? highRiskEntitlements = null,
string? pkgutilIdentifier = null)
{
var metadataBuilder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.OrdinalIgnoreCase);
metadataBuilder["macos:bundle_id"] = bundleId;
metadataBuilder["macos:sandboxed"] = sandboxed ? "true" : "false";
metadataBuilder["macos:hardened_runtime"] = hardenedRuntime ? "true" : "false";
if (!string.IsNullOrWhiteSpace(teamId))
{
metadataBuilder["macos:team_id"] = teamId;
}
if (!string.IsNullOrWhiteSpace(codeResourcesHash))
{
metadataBuilder["macos:code_resources_hash"] = codeResourcesHash;
}
if (categories is not null && categories.Any())
{
metadataBuilder["macos:capability_categories"] = string.Join(",", categories);
}
if (highRiskEntitlements is not null && highRiskEntitlements.Any())
{
metadataBuilder["macos:high_risk_entitlements"] = string.Join(",", highRiskEntitlements);
}
if (!string.IsNullOrWhiteSpace(pkgutilIdentifier))
{
metadataBuilder["pkgutil:identifier"] = pkgutilIdentifier;
}
return new PolicyEvaluationComponent(
bundleId.Split('.').Last(),
version,
"macos-bundle",
$"pkg:generic/macos-app/{bundleId}@{version}",
metadataBuilder.ToImmutable());
}
#region macOS Policy Predicate Tests
private static readonly string MacOsPolicy = """
policy "macOS Security Policy" syntax "stella-dsl@1" {
metadata {
description = "Enforce macOS bundle security requirements."
tags = ["macos","security"]
}
rule block_unsigned_apps priority 3 {
when sbom.any_component(not macos.signed)
then status := "blocked"
because "Unsigned macOS applications are not permitted."
}
rule warn_high_risk_entitlements priority 4 {
when sbom.any_component(macos.high_risk_entitlements)
then warn message "Application requests high-risk entitlements."
because "High-risk entitlements require review."
}
rule require_hardened_runtime priority 5 {
when sbom.any_component(macos.signed and not macos.hardened_runtime)
then warn message "Application does not use hardened runtime."
because "Hardened runtime is recommended for security."
}
rule block_unsandboxed_apps priority 2 {
when sbom.any_component(not macos.sandboxed)
then warn message "Application is not sandboxed."
because "App sandbox provides security isolation."
}
rule block_camera_microphone priority 1 {
when sbom.any_component(macos.category_any(["camera", "microphone"]))
then warn message "Application accesses camera or microphone."
because "Camera/microphone access requires review."
}
}
""";
[Fact]
public void Evaluate_MacOs_UnsignedAppBlocked()
{
var document = compiler.Compile(MacOsPolicy);
Assert.True(document.Success);
var ir = Assert.IsType<PolicyIrDocument>(document.Document);
// Unsigned but sandboxed app to avoid matching the unsandboxed rule first
var component = CreateMacOsComponent(
bundleId: "com.unknown.UnsignedApp",
version: "1.0.0",
sandboxed: true, // Sandboxed to avoid matching block_unsandboxed_apps first
hardenedRuntime: false,
teamId: null,
codeResourcesHash: null);
var context = CreateContext("Medium", "internal") with
{
Sbom = new PolicyEvaluationSbom(
ImmutableHashSet<string>.Empty,
ImmutableArray.Create(component))
};
var result = evaluationService.Evaluate(ir, context);
Assert.True(result.Matched);
Assert.Equal("block_unsigned_apps", result.RuleName);
Assert.Equal("blocked", result.Status);
}
[Fact]
public void Evaluate_MacOs_SignedAppPasses()
{
var document = compiler.Compile(MacOsPolicy);
Assert.True(document.Success);
var ir = Assert.IsType<PolicyIrDocument>(document.Document);
var component = CreateMacOsComponent(
bundleId: "com.apple.Safari",
version: "17.1",
sandboxed: true,
hardenedRuntime: true,
teamId: "APPLE123",
codeResourcesHash: "sha256:abc123");
var context = CreateContext("Medium", "internal") with
{
Sbom = new PolicyEvaluationSbom(
ImmutableHashSet<string>.Empty,
ImmutableArray.Create(component))
};
var result = evaluationService.Evaluate(ir, context);
// No blocking rules should match for a properly signed and sandboxed app
Assert.False(result.Matched && result.Status == "blocked");
}
[Fact]
public void Evaluate_MacOs_HighRiskEntitlementsWarns()
{
var document = compiler.Compile(MacOsPolicy);
Assert.True(document.Success);
var ir = Assert.IsType<PolicyIrDocument>(document.Document);
// App with high-risk entitlements but no camera/microphone categories
// to avoid matching block_camera_microphone rule first
var component = CreateMacOsComponent(
bundleId: "com.example.AutomationApp",
version: "2.0.0",
sandboxed: true,
hardenedRuntime: true,
teamId: "TEAM123",
codeResourcesHash: "sha256:def456",
categories: new[] { "network", "automation" },
highRiskEntitlements: new[] { "com.apple.security.automation.apple-events" });
var context = CreateContext("Medium", "internal") with
{
Sbom = new PolicyEvaluationSbom(
ImmutableHashSet<string>.Empty,
ImmutableArray.Create(component))
};
var result = evaluationService.Evaluate(ir, context);
Assert.True(result.Matched);
Assert.Equal("warn_high_risk_entitlements", result.RuleName);
Assert.Equal("warned", result.Status);
}
[Fact]
public void Evaluate_MacOs_CategoryMatchesCameraAccess()
{
var document = compiler.Compile(MacOsPolicy);
Assert.True(document.Success);
var ir = Assert.IsType<PolicyIrDocument>(document.Document);
var component = CreateMacOsComponent(
bundleId: "com.example.VideoChat",
version: "1.5.0",
sandboxed: true,
hardenedRuntime: true,
teamId: "TEAM456",
codeResourcesHash: "sha256:ghi789",
categories: new[] { "camera", "microphone", "network" });
var context = CreateContext("Medium", "internal") with
{
Sbom = new PolicyEvaluationSbom(
ImmutableHashSet<string>.Empty,
ImmutableArray.Create(component))
};
var result = evaluationService.Evaluate(ir, context);
// Should match camera/microphone warning rule
Assert.True(result.Matched);
// Either high_risk or camera_microphone rule should match
Assert.True(
result.RuleName == "block_camera_microphone" ||
result.RuleName == "warn_high_risk_entitlements" ||
result.Status == "warned");
}
[Fact]
public void Evaluate_MacOs_HardenedRuntimeWarnsWhenMissing()
{
var document = compiler.Compile(MacOsPolicy);
Assert.True(document.Success);
var ir = Assert.IsType<PolicyIrDocument>(document.Document);
var component = CreateMacOsComponent(
bundleId: "com.example.LegacyApp",
version: "3.0.0",
sandboxed: true,
hardenedRuntime: false,
teamId: "TEAM789",
codeResourcesHash: "sha256:jkl012");
var context = CreateContext("Medium", "internal") with
{
Sbom = new PolicyEvaluationSbom(
ImmutableHashSet<string>.Empty,
ImmutableArray.Create(component))
};
var result = evaluationService.Evaluate(ir, context);
Assert.True(result.Matched);
Assert.Equal("require_hardened_runtime", result.RuleName);
Assert.Equal("warned", result.Status);
}
#endregion
}