using System.Collections.Immutable; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using StellaOps.Policy.Engine.Caching; using StellaOps.Policy.Engine.Compilation; using StellaOps.Policy.Engine.Domain; using StellaOps.Policy.Engine.Evaluation; using StellaOps.Policy.Engine.ReachabilityFacts; using StellaOps.Policy.Engine.Options; using StellaOps.Policy.Engine.Services; using StellaOps.Policy.Engine.Signals.Entropy; using StellaOps.Policy.Licensing; using StellaOps.PolicyDsl; using Xunit; using StellaOps.TestKit; namespace StellaOps.Policy.Engine.Tests; public sealed class PolicyRuntimeEvaluationServiceTests { private const string TestPolicy = """ policy "Test Policy" syntax "stella-dsl@1" { rule block_critical priority 10 { when severity.normalized == "Critical" then status := "blocked" because "Block critical findings" } rule warn_high priority 20 { when severity.normalized == "High" then status := "warn" because "Warn on high severity findings" } rule allow_default priority 100 { when true then status := "affected" because "Default affected status" } } """; [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_ReturnsDecisionFromCompiledPolicy() { var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy); var request = CreateRequest("pack-1", 1, severity: "Critical"); var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.Equal("pack-1", response.PackId); Assert.Equal(1, response.Version); Assert.NotNull(response.PolicyDigest); Assert.NotNull(response.Confidence); Assert.False(response.Cached); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_UsesCacheOnSecondCall() { var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy); var request = CreateRequest("pack-1", 1, severity: "High"); // First call - cache miss var response1 = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.False(response1.Cached); // Second call - cache hit var response2 = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.True(response2.Cached); Assert.Equal(CacheSource.InMemory, response2.CacheSource); Assert.Equal(response1.Status, response2.Status); Assert.Equal(response1.CorrelationId, response2.CorrelationId); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_BypassCacheWhenRequested() { var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy); var request = CreateRequest("pack-1", 1, severity: "Medium"); // First call var response1 = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.False(response1.Cached); // Second call with bypass var bypassRequest = request with { BypassCache = true }; var response2 = await harness.Service.EvaluateAsync(bypassRequest, TestContext.Current.CancellationToken); Assert.False(response2.Cached); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_ThrowsOnMissingBundle() { var harness = CreateHarness(); var request = CreateRequest("non-existent", 1, severity: "Low"); await Assert.ThrowsAsync( () => harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken)); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_GeneratesDeterministicCorrelationId() { var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy); var request = CreateRequest("pack-1", 1, severity: "High"); var response1 = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); // Create a new harness with fresh cache var harness2 = CreateHarness(); await harness2.StoreTestPolicyAsync("pack-1", 1, TestPolicy); var response2 = await harness2.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); // Same inputs should produce same correlation ID Assert.Equal(response1.CorrelationId, response2.CorrelationId); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateBatchAsync_ReturnsMultipleResults() { var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy); var requests = new[] { CreateRequest("pack-1", 1, severity: "Critical", subjectPurl: "pkg:npm/lodash@4.17.0"), CreateRequest("pack-1", 1, severity: "High", subjectPurl: "pkg:npm/express@4.18.0"), CreateRequest("pack-1", 1, severity: "Medium", subjectPurl: "pkg:npm/axios@1.0.0"), }; var responses = await harness.Service.EvaluateBatchAsync(requests, TestContext.Current.CancellationToken); Assert.Equal(3, responses.Count); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateBatchAsync_UsesCacheForDuplicates() { var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy); // Pre-populate cache var request = CreateRequest("pack-1", 1, severity: "Critical"); await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); var requests = new[] { request, // Should be cached CreateRequest("pack-1", 1, severity: "High"), // New }; var responses = await harness.Service.EvaluateBatchAsync(requests, TestContext.Current.CancellationToken); Assert.Equal(2, responses.Count); Assert.Contains(responses, r => r.Cached); Assert.Contains(responses, r => !r.Cached); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_DifferentContextsGetDifferentCacheKeys() { var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy); var request1 = CreateRequest("pack-1", 1, severity: "High"); var request2 = CreateRequest("pack-1", 1, severity: "Critical"); var response1 = await harness.Service.EvaluateAsync(request1, TestContext.Current.CancellationToken); var response2 = await harness.Service.EvaluateAsync(request2, TestContext.Current.CancellationToken); // Both should be cache misses (different severity = different context) Assert.False(response1.Cached); Assert.False(response2.Cached); // Different inputs = different correlation IDs Assert.NotEqual(response1.CorrelationId, response2.CorrelationId); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_EnrichesReachabilityFromFacts() { const string policy = """ policy "Reachability policy" syntax "stella-dsl@1" { rule reachable_then_warn priority 5 { when reachability.state == "reachable" then status := "warn" because "reachable path detected" } rule default priority 100 { when true then status := "affected" because "default" } } """; var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-2", 1, policy); var fact = new ReachabilityFact { Id = "fact-1", TenantId = "tenant-1", ComponentPurl = "pkg:npm/lodash@4.17.21", AdvisoryId = "CVE-2024-0001", State = ReachabilityState.Reachable, Confidence = 0.92m, Score = 0.85m, HasRuntimeEvidence = true, Source = "graph-analyzer", Method = AnalysisMethod.Hybrid, EvidenceRef = "evidence/callgraph.json", ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), ExpiresAt = null, Metadata = new Dictionary() }; await harness.ReachabilityStore.SaveAsync(fact, TestContext.Current.CancellationToken); var request = CreateRequest("pack-2", 1, severity: "Low"); var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.Equal("warn", response.Status); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_GatesUnreachableWithoutEvidenceRef_ToUnderInvestigation() { const string policy = """ policy "Reachability gate policy" syntax "stella-dsl@1" { rule unreachable_to_not_affected priority 10 { when reachability.state == "unreachable" then status := "not_affected" because "unreachable + evidence" } rule gated_to_under_investigation priority 20 { when reachability.state == "under_investigation" then status := "under_investigation" because "unreachable but missing evidence" } rule default priority 100 { when true then status := "affected" because "default" } } """; var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-3", 1, policy); var fact = new ReachabilityFact { Id = "fact-1", TenantId = "tenant-1", ComponentPurl = "pkg:npm/lodash@4.17.21", AdvisoryId = "CVE-2024-0001", State = ReachabilityState.Unreachable, Confidence = 0.92m, Score = 0m, HasRuntimeEvidence = false, Source = "graph-analyzer", Method = AnalysisMethod.Static, EvidenceRef = null, EvidenceHash = "sha256:deadbeef", ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), Metadata = new Dictionary() }; await harness.ReachabilityStore.SaveAsync(fact, TestContext.Current.CancellationToken); var request = CreateRequest("pack-3", 1, severity: "Low"); var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.Equal("under_investigation", response.Status); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_GatesUnreachableWithLowConfidence_ToUnderInvestigation() { const string policy = """ policy "Reachability gate policy" syntax "stella-dsl@1" { rule unreachable_to_not_affected priority 10 { when reachability.state == "unreachable" then status := "not_affected" because "unreachable + evidence" } rule gated_to_under_investigation priority 20 { when reachability.state == "under_investigation" then status := "under_investigation" because "unreachable but low confidence" } rule default priority 100 { when true then status := "affected" because "default" } } """; var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-4", 1, policy); var fact = new ReachabilityFact { Id = "fact-1", TenantId = "tenant-1", ComponentPurl = "pkg:npm/lodash@4.17.21", AdvisoryId = "CVE-2024-0001", State = ReachabilityState.Unreachable, Confidence = 0.7m, Score = 0m, HasRuntimeEvidence = false, Source = "graph-analyzer", Method = AnalysisMethod.Static, EvidenceRef = "cas://reachability/facts/fact-1", EvidenceHash = "sha256:deadbeef", ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), Metadata = new Dictionary() }; await harness.ReachabilityStore.SaveAsync(fact, TestContext.Current.CancellationToken); var request = CreateRequest("pack-4", 1, severity: "Low"); var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.Equal("under_investigation", response.Status); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_AllowsUnreachableWithEvidenceRefAndHighConfidence() { const string policy = """ policy "Reachability gate policy" syntax "stella-dsl@1" { rule unreachable_to_not_affected priority 10 { when reachability.state == "unreachable" then status := "not_affected" because "unreachable + evidence" } rule gated_to_under_investigation priority 20 { when reachability.state == "under_investigation" then status := "under_investigation" because "gated" } rule default priority 100 { when true then status := "affected" because "default" } } """; var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-5", 1, policy); var fact = new ReachabilityFact { Id = "fact-1", TenantId = "tenant-1", ComponentPurl = "pkg:npm/lodash@4.17.21", AdvisoryId = "CVE-2024-0001", State = ReachabilityState.Unreachable, Confidence = 0.92m, Score = 0m, HasRuntimeEvidence = false, Source = "graph-analyzer", Method = AnalysisMethod.Static, EvidenceRef = "cas://reachability/facts/fact-1", EvidenceHash = "sha256:deadbeef", ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), Metadata = new Dictionary() }; await harness.ReachabilityStore.SaveAsync(fact, TestContext.Current.CancellationToken); var request = CreateRequest("pack-5", 1, severity: "Low"); var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.Equal("not_affected", response.Status); } [Trait("Category", TestCategories.Unit)] [Fact] public async Task EvaluateAsync_BlocksOnLicenseComplianceFailure() { var harness = CreateHarness(); await harness.StoreTestPolicyAsync("pack-6", 1, TestPolicy); var component = new PolicyEvaluationComponent( Name: "example", Version: "1.0.0", Type: "library", Purl: "pkg:npm/example@1.0.0", Metadata: ImmutableDictionary.Empty.Add("license_expression", "GPL-3.0-only")); var sbom = new PolicyEvaluationSbom( ImmutableHashSet.Empty.WithComparer(StringComparer.OrdinalIgnoreCase), ImmutableArray.Create(component)); var request = CreateRequest("pack-6", 1, severity: "Low", sbom: sbom); var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken); Assert.Equal("blocked", response.Status); Assert.Contains(response.Annotations, pair => pair.Key == "license.status" && pair.Value == "fail"); } private static RuntimeEvaluationRequest CreateRequest( string packId, int version, string severity, string tenantId = "tenant-1", string subjectPurl = "pkg:npm/lodash@4.17.21", string advisoryId = "CVE-2024-0001", PolicyEvaluationSbom? sbom = null) { return new RuntimeEvaluationRequest( packId, version, tenantId, subjectPurl, advisoryId, Severity: new PolicyEvaluationSeverity(severity, null), Advisory: new PolicyEvaluationAdvisory("NVD", ImmutableDictionary.Empty), Vex: PolicyEvaluationVexEvidence.Empty, Sbom: sbom ?? PolicyEvaluationSbom.Empty, Exceptions: PolicyEvaluationExceptions.Empty, Reachability: PolicyEvaluationReachability.Unknown, EntropyLayerSummary: null, EntropyReport: null, ProvenanceAttested: null, EvaluationTimestamp: new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero), BypassCache: false); } private static TestHarness CreateHarness() { var repository = new InMemoryPolicyPackRepository(TimeProvider.System); var cacheLogger = NullLogger.Instance; var serviceLogger = NullLogger.Instance; var options = Microsoft.Extensions.Options.Options.Create(new PolicyEngineOptions()); var cache = new InMemoryPolicyEvaluationCache(cacheLogger, TimeProvider.System, options); var evaluator = new PolicyEvaluator(); var entropy = new EntropyPenaltyCalculator(options, NullLogger.Instance); var licenseOptions = Microsoft.Extensions.Options.Options.Create(new LicenseComplianceOptions { Enabled = true, Policy = LicensePolicyDefaults.Default }); var licenseComplianceService = new LicenseComplianceService( new LicenseComplianceEvaluator(LicenseKnowledgeBase.LoadDefault()), new LicensePolicyLoader(), licenseOptions, NullLogger.Instance); var reachabilityStore = new InMemoryReachabilityFactsStore(TimeProvider.System); var reachabilityCache = new InMemoryReachabilityFactsOverlayCache( NullLogger.Instance, TimeProvider.System, Microsoft.Extensions.Options.Options.Create(new PolicyEngineOptions())); var reachabilityService = new ReachabilityFactsJoiningService( reachabilityStore, reachabilityCache, NullLogger.Instance, TimeProvider.System); var compilationService = CreateCompilationService(); var service = new PolicyRuntimeEvaluationService( repository, cache, evaluator, reachabilityService, entropy, licenseComplianceService, ntiaCompliance: null, TimeProvider.System, serviceLogger); return new TestHarness(service, repository, compilationService, reachabilityStore); } private static PolicyCompilationService CreateCompilationService() { var compiler = new PolicyCompiler(); var analyzer = new PolicyComplexityAnalyzer(); var options = new PolicyEngineOptions(); var optionsMonitor = new StaticOptionsMonitor(options); var metadataExtractor = new PolicyMetadataExtractor(); return new PolicyCompilationService(compiler, analyzer, metadataExtractor, optionsMonitor, TimeProvider.System); } private sealed record TestHarness( PolicyRuntimeEvaluationService Service, InMemoryPolicyPackRepository Repository, PolicyCompilationService CompilationService, InMemoryReachabilityFactsStore ReachabilityStore) { public async Task StoreTestPolicyAsync(string packId, int version, string dsl) { var bundleService = new PolicyBundleService(CompilationService, Repository, TimeProvider.System); var request = new PolicyBundleRequest(new PolicyDslPayload("stella-dsl@1", dsl), SigningKeyId: null); await bundleService.CompileAndStoreAsync(packId, version, request, TestContext.Current.CancellationToken); } } private sealed class StaticOptionsMonitor : IOptionsMonitor { private readonly PolicyEngineOptions _value; public StaticOptionsMonitor(PolicyEngineOptions value) => _value = value; public PolicyEngineOptions CurrentValue => _value; public PolicyEngineOptions Get(string? name) => _value; public IDisposable OnChange(Action listener) => NullDisposable.Instance; private sealed class NullDisposable : IDisposable { public static readonly NullDisposable Instance = new(); public void Dispose() { } } } }