7e384ab610
- Added IsolatedReplayContext class to provide an isolated environment for replaying audit bundles without external calls. - Introduced methods for initializing the context, verifying input digests, and extracting inputs for policy evaluation. - Created supporting interfaces and options for context configuration. feat: Create ReplayExecutor for executing policy re-evaluation and verdict comparison - Developed ReplayExecutor class to handle the execution of replay processes, including input verification and verdict comparison. - Implemented detailed drift detection and error handling during replay execution. - Added interfaces for policy evaluation and replay execution options. feat: Add ScanSnapshotFetcher for fetching scan data and snapshots - Introduced ScanSnapshotFetcher class to retrieve necessary scan data and snapshots for audit bundle creation. - Implemented methods to fetch scan metadata, advisory feeds, policy snapshots, and VEX statements. - Created supporting interfaces for scan data, feed snapshots, and policy snapshots.
402 lines
15 KiB
C#
402 lines
15 KiB
C#
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// Copyright (c) StellaOps
|
|
|
|
using FluentAssertions;
|
|
using StellaOps.Scanner.Explainability.Assumptions;
|
|
using StellaOps.Scanner.Reachability.Stack;
|
|
|
|
namespace StellaOps.Scanner.Reachability.Stack.Tests;
|
|
|
|
public class ReachabilityStackEvaluatorTests
|
|
{
|
|
private readonly ReachabilityStackEvaluator _evaluator = new();
|
|
|
|
private static VulnerableSymbol CreateTestSymbol() => new(
|
|
Name: "EVP_DecryptUpdate",
|
|
Library: "libcrypto.so.1.1",
|
|
Version: "1.1.1",
|
|
VulnerabilityId: "CVE-2024-1234",
|
|
Type: SymbolType.Function
|
|
);
|
|
|
|
private static ReachabilityLayer1 CreateLayer1(bool isReachable, ConfidenceLevel confidence) => new()
|
|
{
|
|
IsReachable = isReachable,
|
|
Confidence = confidence,
|
|
AnalysisMethod = "Static call graph"
|
|
};
|
|
|
|
private static ReachabilityLayer2 CreateLayer2(bool isResolved, ConfidenceLevel confidence) => new()
|
|
{
|
|
IsResolved = isResolved,
|
|
Confidence = confidence,
|
|
Reason = isResolved ? "Symbol found in linked library" : "Symbol not linked"
|
|
};
|
|
|
|
private static ReachabilityLayer3 CreateLayer3(bool isGated, GatingOutcome outcome, ConfidenceLevel confidence) => new()
|
|
{
|
|
IsGated = isGated,
|
|
Outcome = outcome,
|
|
Confidence = confidence
|
|
};
|
|
|
|
#region Verdict Truth Table Tests
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_AllThreeConfirmReachable_ReturnsExploitable()
|
|
{
|
|
// L1=Reachable, L2=Resolved, L3=NotGated -> Exploitable
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.Exploitable);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_L1L2ConfirmL3Unknown_ReturnsLikelyExploitable()
|
|
{
|
|
// L1=Reachable, L2=Resolved, L3=Unknown -> LikelyExploitable
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.Unknown, ConfidenceLevel.Low);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.LikelyExploitable);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_L1L2ConfirmL3Conditional_ReturnsLikelyExploitable()
|
|
{
|
|
// L1=Reachable, L2=Resolved, L3=Conditional -> LikelyExploitable
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: true, GatingOutcome.Conditional, ConfidenceLevel.Medium);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.LikelyExploitable);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_L1ReachableL2NotResolved_ReturnsUnreachable()
|
|
{
|
|
// L1=Reachable, L2=NotResolved (confirmed) -> Unreachable
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: false, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.Unreachable);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_L1NotReachable_ReturnsUnreachable()
|
|
{
|
|
// L1=NotReachable (confirmed) -> Unreachable
|
|
var layer1 = CreateLayer1(isReachable: false, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.Unreachable);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_L3Blocked_ReturnsUnreachable()
|
|
{
|
|
// L1=Reachable, L2=Resolved, L3=Blocked (confirmed) -> Unreachable
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: true, GatingOutcome.Blocked, ConfidenceLevel.High);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.Unreachable);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_L1ReachableL2LowConfidence_ReturnsPossiblyExploitable()
|
|
{
|
|
// L1=Reachable, L2=Unknown (low confidence) -> PossiblyExploitable
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: false, ConfidenceLevel.Low);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.Unknown, ConfidenceLevel.Low);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.PossiblyExploitable);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_L1LowConfidenceNoData_ReturnsUnknown()
|
|
{
|
|
// L1=Unknown (low confidence, no paths) -> Unknown
|
|
var layer1 = new ReachabilityLayer1
|
|
{
|
|
IsReachable = false,
|
|
Confidence = ConfidenceLevel.Low,
|
|
Paths = []
|
|
};
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.Unknown);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Evaluate Tests
|
|
|
|
[Fact]
|
|
public void Evaluate_CreatesCompleteStack()
|
|
{
|
|
var symbol = CreateTestSymbol();
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
|
|
|
var stack = _evaluator.Evaluate("finding-123", symbol, layer1, layer2, layer3);
|
|
|
|
stack.Id.Should().NotBeNullOrEmpty();
|
|
stack.FindingId.Should().Be("finding-123");
|
|
stack.Symbol.Should().Be(symbol);
|
|
stack.StaticCallGraph.Should().Be(layer1);
|
|
stack.BinaryResolution.Should().Be(layer2);
|
|
stack.RuntimeGating.Should().Be(layer3);
|
|
stack.Verdict.Should().Be(ReachabilityVerdict.Exploitable);
|
|
stack.AnalyzedAt.Should().BeCloseTo(DateTimeOffset.UtcNow, TimeSpan.FromSeconds(5));
|
|
stack.Explanation.Should().NotBeNullOrEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void Evaluate_ExploitableVerdict_ExplanationContainsAllThreeLayers()
|
|
{
|
|
var symbol = CreateTestSymbol();
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
|
|
|
var stack = _evaluator.Evaluate("finding-123", symbol, layer1, layer2, layer3);
|
|
|
|
stack.Explanation.Should().Contain("Layer 1");
|
|
stack.Explanation.Should().Contain("Layer 2");
|
|
stack.Explanation.Should().Contain("Layer 3");
|
|
stack.Explanation.Should().Contain("exploitable");
|
|
}
|
|
|
|
[Fact]
|
|
public void Evaluate_UnreachableVerdict_ExplanationMentionsBlocking()
|
|
{
|
|
var symbol = CreateTestSymbol();
|
|
var layer1 = CreateLayer1(isReachable: false, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
|
|
|
var stack = _evaluator.Evaluate("finding-123", symbol, layer1, layer2, layer3);
|
|
|
|
stack.Verdict.Should().Be(ReachabilityVerdict.Unreachable);
|
|
stack.Explanation.Should().Contain("block");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Model Tests
|
|
|
|
[Fact]
|
|
public void VulnerableSymbol_StoresAllProperties()
|
|
{
|
|
var symbol = new VulnerableSymbol(
|
|
Name: "vulnerable_function",
|
|
Library: "libvuln.so",
|
|
Version: "2.0.0",
|
|
VulnerabilityId: "CVE-2024-5678",
|
|
Type: SymbolType.Function
|
|
);
|
|
|
|
symbol.Name.Should().Be("vulnerable_function");
|
|
symbol.Library.Should().Be("libvuln.so");
|
|
symbol.Version.Should().Be("2.0.0");
|
|
symbol.VulnerabilityId.Should().Be("CVE-2024-5678");
|
|
symbol.Type.Should().Be(SymbolType.Function);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(SymbolType.Function)]
|
|
[InlineData(SymbolType.Method)]
|
|
[InlineData(SymbolType.JavaMethod)]
|
|
[InlineData(SymbolType.JsFunction)]
|
|
[InlineData(SymbolType.PyFunction)]
|
|
[InlineData(SymbolType.GoFunction)]
|
|
[InlineData(SymbolType.RustFunction)]
|
|
public void SymbolType_AllValuesAreValid(SymbolType type)
|
|
{
|
|
var symbol = new VulnerableSymbol("test", null, null, "CVE-1234", type);
|
|
symbol.Type.Should().Be(type);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(ReachabilityVerdict.Exploitable)]
|
|
[InlineData(ReachabilityVerdict.LikelyExploitable)]
|
|
[InlineData(ReachabilityVerdict.PossiblyExploitable)]
|
|
[InlineData(ReachabilityVerdict.Unreachable)]
|
|
[InlineData(ReachabilityVerdict.Unknown)]
|
|
public void ReachabilityVerdict_AllValuesAreValid(ReachabilityVerdict verdict)
|
|
{
|
|
// Verify enum value is defined
|
|
Enum.IsDefined(typeof(ReachabilityVerdict), verdict).Should().BeTrue();
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(GatingOutcome.NotGated)]
|
|
[InlineData(GatingOutcome.Blocked)]
|
|
[InlineData(GatingOutcome.Conditional)]
|
|
[InlineData(GatingOutcome.Unknown)]
|
|
public void GatingOutcome_AllValuesAreValid(GatingOutcome outcome)
|
|
{
|
|
var layer3 = CreateLayer3(isGated: false, outcome, ConfidenceLevel.Medium);
|
|
layer3.Outcome.Should().Be(outcome);
|
|
}
|
|
|
|
[Fact]
|
|
public void GatingCondition_StoresAllProperties()
|
|
{
|
|
var condition = new GatingCondition(
|
|
Type: GatingType.FeatureFlag,
|
|
Description: "Feature flag check",
|
|
ConfigKey: "feature.enabled",
|
|
EnvVar: null,
|
|
IsBlocking: true,
|
|
Status: GatingStatus.Disabled
|
|
);
|
|
|
|
condition.Type.Should().Be(GatingType.FeatureFlag);
|
|
condition.Description.Should().Be("Feature flag check");
|
|
condition.ConfigKey.Should().Be("feature.enabled");
|
|
condition.IsBlocking.Should().BeTrue();
|
|
condition.Status.Should().Be(GatingStatus.Disabled);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(GatingType.FeatureFlag)]
|
|
[InlineData(GatingType.EnvironmentVariable)]
|
|
[InlineData(GatingType.ConfigurationValue)]
|
|
[InlineData(GatingType.CompileTimeConditional)]
|
|
[InlineData(GatingType.PlatformCheck)]
|
|
[InlineData(GatingType.CapabilityCheck)]
|
|
[InlineData(GatingType.LicenseCheck)]
|
|
[InlineData(GatingType.ExperimentFlag)]
|
|
public void GatingType_AllValuesAreValid(GatingType type)
|
|
{
|
|
var condition = new GatingCondition(type, "test", null, null, false, GatingStatus.Unknown);
|
|
condition.Type.Should().Be(type);
|
|
}
|
|
|
|
[Fact]
|
|
public void CallPath_WithSites_StoresCorrectly()
|
|
{
|
|
var entrypoint = new Entrypoint("Main", EntrypointType.Main, "Program.cs", "Application entry");
|
|
var sites = new[]
|
|
{
|
|
new CallSite("Main", "Program", "Program.cs", 10, CallSiteType.Direct),
|
|
new CallSite("ProcessData", "DataService", "DataService.cs", 45, CallSiteType.Virtual),
|
|
new CallSite("vulnerable_function", null, "native.c", null, CallSiteType.Dynamic)
|
|
};
|
|
|
|
var path = new CallPath
|
|
{
|
|
Sites = [.. sites],
|
|
Entrypoint = entrypoint,
|
|
Confidence = 0.85,
|
|
HasConditionals = true
|
|
};
|
|
|
|
path.Sites.Should().HaveCount(3);
|
|
path.Entrypoint.Should().Be(entrypoint);
|
|
path.Confidence.Should().Be(0.85);
|
|
path.HasConditionals.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void SymbolResolution_StoresDetails()
|
|
{
|
|
var resolution = new SymbolResolution(
|
|
SymbolName: "EVP_DecryptUpdate",
|
|
ResolvedLibrary: "/usr/lib/libcrypto.so.1.1",
|
|
ResolvedVersion: "1.1.1k",
|
|
SymbolVersion: "OPENSSL_1_1_0",
|
|
Method: ResolutionMethod.DirectLink
|
|
);
|
|
|
|
resolution.SymbolName.Should().Be("EVP_DecryptUpdate");
|
|
resolution.ResolvedLibrary.Should().Be("/usr/lib/libcrypto.so.1.1");
|
|
resolution.SymbolVersion.Should().Be("OPENSSL_1_1_0");
|
|
resolution.Method.Should().Be(ResolutionMethod.DirectLink);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(ResolutionMethod.DirectLink)]
|
|
[InlineData(ResolutionMethod.DynamicLoad)]
|
|
[InlineData(ResolutionMethod.DelayLoad)]
|
|
[InlineData(ResolutionMethod.WeakSymbol)]
|
|
[InlineData(ResolutionMethod.Interposition)]
|
|
public void ResolutionMethod_AllValuesAreValid(ResolutionMethod method)
|
|
{
|
|
var resolution = new SymbolResolution("sym", "lib", null, null, method);
|
|
resolution.Method.Should().Be(method);
|
|
}
|
|
|
|
[Fact]
|
|
public void LoaderRule_StoresProperties()
|
|
{
|
|
var rule = new LoaderRule(
|
|
Type: LoaderRuleType.Rpath,
|
|
Value: "/opt/myapp/lib",
|
|
Source: "ELF binary"
|
|
);
|
|
|
|
rule.Type.Should().Be(LoaderRuleType.Rpath);
|
|
rule.Value.Should().Be("/opt/myapp/lib");
|
|
rule.Source.Should().Be("ELF binary");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Edge Case Tests
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_L3BlockedButLowConfidence_DoesNotBlock()
|
|
{
|
|
// L3 blocked but low confidence should not definitively block
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
|
var layer3 = CreateLayer3(isGated: true, GatingOutcome.Blocked, ConfidenceLevel.Low);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
// With low confidence blocking, should still be exploitable since we can't trust the block
|
|
verdict.Should().Be(ReachabilityVerdict.Exploitable);
|
|
}
|
|
|
|
[Fact]
|
|
public void DeriveVerdict_AllLayersHighConfidence_ExploitableIsDefinitive()
|
|
{
|
|
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.Verified);
|
|
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.Verified);
|
|
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.Verified);
|
|
|
|
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
|
|
|
verdict.Should().Be(ReachabilityVerdict.Exploitable);
|
|
}
|
|
|
|
#endregion
|
|
}
|