553 lines
21 KiB
C#
553 lines
21 KiB
C#
using System.Collections.Immutable;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using StellaOps.Policy.Engine.Caching;
|
|
using StellaOps.Policy.Engine.Compilation;
|
|
using StellaOps.Policy.Engine.Domain;
|
|
using StellaOps.Policy.Engine.Evaluation;
|
|
using StellaOps.Policy.Engine.ReachabilityFacts;
|
|
using StellaOps.Policy.Engine.Options;
|
|
using StellaOps.Policy.Engine.Services;
|
|
using StellaOps.Policy.Engine.Signals.Entropy;
|
|
using StellaOps.Policy.Licensing;
|
|
using StellaOps.PolicyDsl;
|
|
using Xunit;
|
|
|
|
using StellaOps.TestKit;
|
|
namespace StellaOps.Policy.Engine.Tests;
|
|
|
|
public sealed class PolicyRuntimeEvaluationServiceTests
|
|
{
|
|
private const string TestPolicy = """
|
|
policy "Test Policy" syntax "stella-dsl@1" {
|
|
rule block_critical priority 10 {
|
|
when severity.normalized == "Critical"
|
|
then status := "blocked"
|
|
because "Block critical findings"
|
|
}
|
|
|
|
rule warn_high priority 20 {
|
|
when severity.normalized == "High"
|
|
then status := "warn"
|
|
because "Warn on high severity findings"
|
|
}
|
|
|
|
rule allow_default priority 100 {
|
|
when true
|
|
then status := "affected"
|
|
because "Default affected status"
|
|
}
|
|
}
|
|
""";
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_ReturnsDecisionFromCompiledPolicy()
|
|
{
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy);
|
|
|
|
var request = CreateRequest("pack-1", 1, severity: "Critical");
|
|
|
|
var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
Assert.Equal("pack-1", response.PackId);
|
|
Assert.Equal(1, response.Version);
|
|
Assert.NotNull(response.PolicyDigest);
|
|
Assert.NotNull(response.Confidence);
|
|
Assert.False(response.Cached);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_UsesCacheOnSecondCall()
|
|
{
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy);
|
|
|
|
var request = CreateRequest("pack-1", 1, severity: "High");
|
|
|
|
// First call - cache miss
|
|
var response1 = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
Assert.False(response1.Cached);
|
|
|
|
// Second call - cache hit
|
|
var response2 = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
Assert.True(response2.Cached);
|
|
Assert.Equal(CacheSource.InMemory, response2.CacheSource);
|
|
Assert.Equal(response1.Status, response2.Status);
|
|
Assert.Equal(response1.CorrelationId, response2.CorrelationId);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_BypassCacheWhenRequested()
|
|
{
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy);
|
|
|
|
var request = CreateRequest("pack-1", 1, severity: "Medium");
|
|
|
|
// First call
|
|
var response1 = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
Assert.False(response1.Cached);
|
|
|
|
// Second call with bypass
|
|
var bypassRequest = request with { BypassCache = true };
|
|
var response2 = await harness.Service.EvaluateAsync(bypassRequest, TestContext.Current.CancellationToken);
|
|
Assert.False(response2.Cached);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_ThrowsOnMissingBundle()
|
|
{
|
|
var harness = CreateHarness();
|
|
var request = CreateRequest("non-existent", 1, severity: "Low");
|
|
|
|
await Assert.ThrowsAsync<InvalidOperationException>(
|
|
() => harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken));
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_GeneratesDeterministicCorrelationId()
|
|
{
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy);
|
|
|
|
var request = CreateRequest("pack-1", 1, severity: "High");
|
|
|
|
var response1 = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
// Create a new harness with fresh cache
|
|
var harness2 = CreateHarness();
|
|
await harness2.StoreTestPolicyAsync("pack-1", 1, TestPolicy);
|
|
|
|
var response2 = await harness2.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
// Same inputs should produce same correlation ID
|
|
Assert.Equal(response1.CorrelationId, response2.CorrelationId);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateBatchAsync_ReturnsMultipleResults()
|
|
{
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy);
|
|
|
|
var requests = new[]
|
|
{
|
|
CreateRequest("pack-1", 1, severity: "Critical", subjectPurl: "pkg:npm/lodash@4.17.0"),
|
|
CreateRequest("pack-1", 1, severity: "High", subjectPurl: "pkg:npm/express@4.18.0"),
|
|
CreateRequest("pack-1", 1, severity: "Medium", subjectPurl: "pkg:npm/axios@1.0.0"),
|
|
};
|
|
|
|
var responses = await harness.Service.EvaluateBatchAsync(requests, TestContext.Current.CancellationToken);
|
|
|
|
Assert.Equal(3, responses.Count);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateBatchAsync_UsesCacheForDuplicates()
|
|
{
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy);
|
|
|
|
// Pre-populate cache
|
|
var request = CreateRequest("pack-1", 1, severity: "Critical");
|
|
await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
var requests = new[]
|
|
{
|
|
request, // Should be cached
|
|
CreateRequest("pack-1", 1, severity: "High"), // New
|
|
};
|
|
|
|
var responses = await harness.Service.EvaluateBatchAsync(requests, TestContext.Current.CancellationToken);
|
|
|
|
Assert.Equal(2, responses.Count);
|
|
Assert.Contains(responses, r => r.Cached);
|
|
Assert.Contains(responses, r => !r.Cached);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_DifferentContextsGetDifferentCacheKeys()
|
|
{
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-1", 1, TestPolicy);
|
|
|
|
var request1 = CreateRequest("pack-1", 1, severity: "High");
|
|
var request2 = CreateRequest("pack-1", 1, severity: "Critical");
|
|
|
|
var response1 = await harness.Service.EvaluateAsync(request1, TestContext.Current.CancellationToken);
|
|
var response2 = await harness.Service.EvaluateAsync(request2, TestContext.Current.CancellationToken);
|
|
|
|
// Both should be cache misses (different severity = different context)
|
|
Assert.False(response1.Cached);
|
|
Assert.False(response2.Cached);
|
|
// Different inputs = different correlation IDs
|
|
Assert.NotEqual(response1.CorrelationId, response2.CorrelationId);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_EnrichesReachabilityFromFacts()
|
|
{
|
|
const string policy = """
|
|
policy "Reachability policy" syntax "stella-dsl@1" {
|
|
rule reachable_then_warn priority 5 {
|
|
when reachability.state == "reachable"
|
|
then status := "warn"
|
|
because "reachable path detected"
|
|
}
|
|
|
|
rule default priority 100 {
|
|
when true
|
|
then status := "affected"
|
|
because "default"
|
|
}
|
|
}
|
|
""";
|
|
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-2", 1, policy);
|
|
|
|
var fact = new ReachabilityFact
|
|
{
|
|
Id = "fact-1",
|
|
TenantId = "tenant-1",
|
|
ComponentPurl = "pkg:npm/lodash@4.17.21",
|
|
AdvisoryId = "CVE-2024-0001",
|
|
State = ReachabilityState.Reachable,
|
|
Confidence = 0.92m,
|
|
Score = 0.85m,
|
|
HasRuntimeEvidence = true,
|
|
Source = "graph-analyzer",
|
|
Method = AnalysisMethod.Hybrid,
|
|
EvidenceRef = "evidence/callgraph.json",
|
|
ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
|
ExpiresAt = null,
|
|
Metadata = new Dictionary<string, object?>()
|
|
};
|
|
|
|
await harness.ReachabilityStore.SaveAsync(fact, TestContext.Current.CancellationToken);
|
|
|
|
var request = CreateRequest("pack-2", 1, severity: "Low");
|
|
|
|
var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
Assert.Equal("warn", response.Status);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_GatesUnreachableWithoutEvidenceRef_ToUnderInvestigation()
|
|
{
|
|
const string policy = """
|
|
policy "Reachability gate policy" syntax "stella-dsl@1" {
|
|
rule unreachable_to_not_affected priority 10 {
|
|
when reachability.state == "unreachable"
|
|
then status := "not_affected"
|
|
because "unreachable + evidence"
|
|
}
|
|
|
|
rule gated_to_under_investigation priority 20 {
|
|
when reachability.state == "under_investigation"
|
|
then status := "under_investigation"
|
|
because "unreachable but missing evidence"
|
|
}
|
|
|
|
rule default priority 100 {
|
|
when true
|
|
then status := "affected"
|
|
because "default"
|
|
}
|
|
}
|
|
""";
|
|
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-3", 1, policy);
|
|
|
|
var fact = new ReachabilityFact
|
|
{
|
|
Id = "fact-1",
|
|
TenantId = "tenant-1",
|
|
ComponentPurl = "pkg:npm/lodash@4.17.21",
|
|
AdvisoryId = "CVE-2024-0001",
|
|
State = ReachabilityState.Unreachable,
|
|
Confidence = 0.92m,
|
|
Score = 0m,
|
|
HasRuntimeEvidence = false,
|
|
Source = "graph-analyzer",
|
|
Method = AnalysisMethod.Static,
|
|
EvidenceRef = null,
|
|
EvidenceHash = "sha256:deadbeef",
|
|
ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
|
Metadata = new Dictionary<string, object?>()
|
|
};
|
|
|
|
await harness.ReachabilityStore.SaveAsync(fact, TestContext.Current.CancellationToken);
|
|
|
|
var request = CreateRequest("pack-3", 1, severity: "Low");
|
|
var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
Assert.Equal("under_investigation", response.Status);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_GatesUnreachableWithLowConfidence_ToUnderInvestigation()
|
|
{
|
|
const string policy = """
|
|
policy "Reachability gate policy" syntax "stella-dsl@1" {
|
|
rule unreachable_to_not_affected priority 10 {
|
|
when reachability.state == "unreachable"
|
|
then status := "not_affected"
|
|
because "unreachable + evidence"
|
|
}
|
|
|
|
rule gated_to_under_investigation priority 20 {
|
|
when reachability.state == "under_investigation"
|
|
then status := "under_investigation"
|
|
because "unreachable but low confidence"
|
|
}
|
|
|
|
rule default priority 100 {
|
|
when true
|
|
then status := "affected"
|
|
because "default"
|
|
}
|
|
}
|
|
""";
|
|
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-4", 1, policy);
|
|
|
|
var fact = new ReachabilityFact
|
|
{
|
|
Id = "fact-1",
|
|
TenantId = "tenant-1",
|
|
ComponentPurl = "pkg:npm/lodash@4.17.21",
|
|
AdvisoryId = "CVE-2024-0001",
|
|
State = ReachabilityState.Unreachable,
|
|
Confidence = 0.7m,
|
|
Score = 0m,
|
|
HasRuntimeEvidence = false,
|
|
Source = "graph-analyzer",
|
|
Method = AnalysisMethod.Static,
|
|
EvidenceRef = "cas://reachability/facts/fact-1",
|
|
EvidenceHash = "sha256:deadbeef",
|
|
ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
|
Metadata = new Dictionary<string, object?>()
|
|
};
|
|
|
|
await harness.ReachabilityStore.SaveAsync(fact, TestContext.Current.CancellationToken);
|
|
|
|
var request = CreateRequest("pack-4", 1, severity: "Low");
|
|
var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
Assert.Equal("under_investigation", response.Status);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_AllowsUnreachableWithEvidenceRefAndHighConfidence()
|
|
{
|
|
const string policy = """
|
|
policy "Reachability gate policy" syntax "stella-dsl@1" {
|
|
rule unreachable_to_not_affected priority 10 {
|
|
when reachability.state == "unreachable"
|
|
then status := "not_affected"
|
|
because "unreachable + evidence"
|
|
}
|
|
|
|
rule gated_to_under_investigation priority 20 {
|
|
when reachability.state == "under_investigation"
|
|
then status := "under_investigation"
|
|
because "gated"
|
|
}
|
|
|
|
rule default priority 100 {
|
|
when true
|
|
then status := "affected"
|
|
because "default"
|
|
}
|
|
}
|
|
""";
|
|
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-5", 1, policy);
|
|
|
|
var fact = new ReachabilityFact
|
|
{
|
|
Id = "fact-1",
|
|
TenantId = "tenant-1",
|
|
ComponentPurl = "pkg:npm/lodash@4.17.21",
|
|
AdvisoryId = "CVE-2024-0001",
|
|
State = ReachabilityState.Unreachable,
|
|
Confidence = 0.92m,
|
|
Score = 0m,
|
|
HasRuntimeEvidence = false,
|
|
Source = "graph-analyzer",
|
|
Method = AnalysisMethod.Static,
|
|
EvidenceRef = "cas://reachability/facts/fact-1",
|
|
EvidenceHash = "sha256:deadbeef",
|
|
ComputedAt = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
|
Metadata = new Dictionary<string, object?>()
|
|
};
|
|
|
|
await harness.ReachabilityStore.SaveAsync(fact, TestContext.Current.CancellationToken);
|
|
|
|
var request = CreateRequest("pack-5", 1, severity: "Low");
|
|
var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
Assert.Equal("not_affected", response.Status);
|
|
}
|
|
|
|
[Trait("Category", TestCategories.Unit)]
|
|
[Fact]
|
|
public async Task EvaluateAsync_BlocksOnLicenseComplianceFailure()
|
|
{
|
|
var harness = CreateHarness();
|
|
await harness.StoreTestPolicyAsync("pack-6", 1, TestPolicy);
|
|
|
|
var component = new PolicyEvaluationComponent(
|
|
Name: "example",
|
|
Version: "1.0.0",
|
|
Type: "library",
|
|
Purl: "pkg:npm/example@1.0.0",
|
|
Metadata: ImmutableDictionary<string, string>.Empty.Add("license_expression", "GPL-3.0-only"));
|
|
var sbom = new PolicyEvaluationSbom(
|
|
ImmutableHashSet<string>.Empty.WithComparer(StringComparer.OrdinalIgnoreCase),
|
|
ImmutableArray.Create(component));
|
|
|
|
var request = CreateRequest("pack-6", 1, severity: "Low", sbom: sbom);
|
|
var response = await harness.Service.EvaluateAsync(request, TestContext.Current.CancellationToken);
|
|
|
|
Assert.Equal("blocked", response.Status);
|
|
Assert.Contains(response.Annotations, pair => pair.Key == "license.status" && pair.Value == "fail");
|
|
}
|
|
|
|
private static RuntimeEvaluationRequest CreateRequest(
|
|
string packId,
|
|
int version,
|
|
string severity,
|
|
string tenantId = "tenant-1",
|
|
string subjectPurl = "pkg:npm/lodash@4.17.21",
|
|
string advisoryId = "CVE-2024-0001",
|
|
PolicyEvaluationSbom? sbom = null)
|
|
{
|
|
return new RuntimeEvaluationRequest(
|
|
packId,
|
|
version,
|
|
tenantId,
|
|
subjectPurl,
|
|
advisoryId,
|
|
Severity: new PolicyEvaluationSeverity(severity, null),
|
|
Advisory: new PolicyEvaluationAdvisory("NVD", ImmutableDictionary<string, string>.Empty),
|
|
Vex: PolicyEvaluationVexEvidence.Empty,
|
|
Sbom: sbom ?? PolicyEvaluationSbom.Empty,
|
|
Exceptions: PolicyEvaluationExceptions.Empty,
|
|
Reachability: PolicyEvaluationReachability.Unknown,
|
|
EntropyLayerSummary: null,
|
|
EntropyReport: null,
|
|
ProvenanceAttested: null,
|
|
EvaluationTimestamp: new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero),
|
|
BypassCache: false);
|
|
}
|
|
|
|
private static TestHarness CreateHarness()
|
|
{
|
|
var repository = new InMemoryPolicyPackRepository(TimeProvider.System);
|
|
var cacheLogger = NullLogger<InMemoryPolicyEvaluationCache>.Instance;
|
|
var serviceLogger = NullLogger<PolicyRuntimeEvaluationService>.Instance;
|
|
var options = Microsoft.Extensions.Options.Options.Create(new PolicyEngineOptions());
|
|
var cache = new InMemoryPolicyEvaluationCache(cacheLogger, TimeProvider.System, options);
|
|
var evaluator = new PolicyEvaluator();
|
|
var entropy = new EntropyPenaltyCalculator(options, NullLogger<EntropyPenaltyCalculator>.Instance);
|
|
var licenseOptions = Microsoft.Extensions.Options.Options.Create(new LicenseComplianceOptions
|
|
{
|
|
Enabled = true,
|
|
Policy = LicensePolicyDefaults.Default
|
|
});
|
|
var licenseComplianceService = new LicenseComplianceService(
|
|
new LicenseComplianceEvaluator(LicenseKnowledgeBase.LoadDefault()),
|
|
new LicensePolicyLoader(),
|
|
licenseOptions,
|
|
NullLogger<LicenseComplianceService>.Instance);
|
|
|
|
var reachabilityStore = new InMemoryReachabilityFactsStore(TimeProvider.System);
|
|
var reachabilityCache = new InMemoryReachabilityFactsOverlayCache(
|
|
NullLogger<InMemoryReachabilityFactsOverlayCache>.Instance,
|
|
TimeProvider.System,
|
|
Microsoft.Extensions.Options.Options.Create(new PolicyEngineOptions()));
|
|
var reachabilityService = new ReachabilityFactsJoiningService(
|
|
reachabilityStore,
|
|
reachabilityCache,
|
|
NullLogger<ReachabilityFactsJoiningService>.Instance,
|
|
TimeProvider.System);
|
|
|
|
var compilationService = CreateCompilationService();
|
|
|
|
var service = new PolicyRuntimeEvaluationService(
|
|
repository,
|
|
cache,
|
|
evaluator,
|
|
reachabilityService,
|
|
entropy,
|
|
licenseComplianceService,
|
|
ntiaCompliance: null,
|
|
TimeProvider.System,
|
|
serviceLogger);
|
|
|
|
return new TestHarness(service, repository, compilationService, reachabilityStore);
|
|
}
|
|
|
|
private static PolicyCompilationService CreateCompilationService()
|
|
{
|
|
var compiler = new PolicyCompiler();
|
|
var analyzer = new PolicyComplexityAnalyzer();
|
|
var options = new PolicyEngineOptions();
|
|
var optionsMonitor = new StaticOptionsMonitor(options);
|
|
var metadataExtractor = new PolicyMetadataExtractor();
|
|
return new PolicyCompilationService(compiler, analyzer, metadataExtractor, optionsMonitor, TimeProvider.System);
|
|
}
|
|
|
|
private sealed record TestHarness(
|
|
PolicyRuntimeEvaluationService Service,
|
|
InMemoryPolicyPackRepository Repository,
|
|
PolicyCompilationService CompilationService,
|
|
InMemoryReachabilityFactsStore ReachabilityStore)
|
|
{
|
|
public async Task StoreTestPolicyAsync(string packId, int version, string dsl)
|
|
{
|
|
var bundleService = new PolicyBundleService(CompilationService, Repository, TimeProvider.System);
|
|
var request = new PolicyBundleRequest(new PolicyDslPayload("stella-dsl@1", dsl), SigningKeyId: null);
|
|
await bundleService.CompileAndStoreAsync(packId, version, request, TestContext.Current.CancellationToken);
|
|
}
|
|
}
|
|
|
|
private sealed class StaticOptionsMonitor : IOptionsMonitor<PolicyEngineOptions>
|
|
{
|
|
private readonly PolicyEngineOptions _value;
|
|
|
|
public StaticOptionsMonitor(PolicyEngineOptions value) => _value = value;
|
|
|
|
public PolicyEngineOptions CurrentValue => _value;
|
|
|
|
public PolicyEngineOptions Get(string? name) => _value;
|
|
|
|
public IDisposable OnChange(Action<PolicyEngineOptions, string> listener) => NullDisposable.Instance;
|
|
|
|
private sealed class NullDisposable : IDisposable
|
|
{
|
|
public static readonly NullDisposable Instance = new();
|
|
public void Dispose() { }
|
|
}
|
|
}
|
|
}
|