feat: Implement IsolatedReplayContext for deterministic audit replay
- Added IsolatedReplayContext class to provide an isolated environment for replaying audit bundles without external calls. - Introduced methods for initializing the context, verifying input digests, and extracting inputs for policy evaluation. - Created supporting interfaces and options for context configuration. feat: Create ReplayExecutor for executing policy re-evaluation and verdict comparison - Developed ReplayExecutor class to handle the execution of replay processes, including input verification and verdict comparison. - Implemented detailed drift detection and error handling during replay execution. - Added interfaces for policy evaluation and replay execution options. feat: Add ScanSnapshotFetcher for fetching scan data and snapshots - Introduced ScanSnapshotFetcher class to retrieve necessary scan data and snapshots for audit bundle creation. - Implemented methods to fetch scan metadata, advisory feeds, policy snapshots, and VEX statements. - Created supporting interfaces for scan data, feed snapshots, and policy snapshots.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,401 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Copyright (c) StellaOps
|
||||
|
||||
using FluentAssertions;
|
||||
using StellaOps.Scanner.Explainability.Assumptions;
|
||||
using StellaOps.Scanner.Reachability.Stack;
|
||||
|
||||
namespace StellaOps.Scanner.Reachability.Stack.Tests;
|
||||
|
||||
public class ReachabilityStackEvaluatorTests
|
||||
{
|
||||
private readonly ReachabilityStackEvaluator _evaluator = new();
|
||||
|
||||
private static VulnerableSymbol CreateTestSymbol() => new(
|
||||
Name: "EVP_DecryptUpdate",
|
||||
Library: "libcrypto.so.1.1",
|
||||
Version: "1.1.1",
|
||||
VulnerabilityId: "CVE-2024-1234",
|
||||
Type: SymbolType.Function
|
||||
);
|
||||
|
||||
private static ReachabilityLayer1 CreateLayer1(bool isReachable, ConfidenceLevel confidence) => new()
|
||||
{
|
||||
IsReachable = isReachable,
|
||||
Confidence = confidence,
|
||||
AnalysisMethod = "Static call graph"
|
||||
};
|
||||
|
||||
private static ReachabilityLayer2 CreateLayer2(bool isResolved, ConfidenceLevel confidence) => new()
|
||||
{
|
||||
IsResolved = isResolved,
|
||||
Confidence = confidence,
|
||||
Reason = isResolved ? "Symbol found in linked library" : "Symbol not linked"
|
||||
};
|
||||
|
||||
private static ReachabilityLayer3 CreateLayer3(bool isGated, GatingOutcome outcome, ConfidenceLevel confidence) => new()
|
||||
{
|
||||
IsGated = isGated,
|
||||
Outcome = outcome,
|
||||
Confidence = confidence
|
||||
};
|
||||
|
||||
#region Verdict Truth Table Tests
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_AllThreeConfirmReachable_ReturnsExploitable()
|
||||
{
|
||||
// L1=Reachable, L2=Resolved, L3=NotGated -> Exploitable
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.Exploitable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_L1L2ConfirmL3Unknown_ReturnsLikelyExploitable()
|
||||
{
|
||||
// L1=Reachable, L2=Resolved, L3=Unknown -> LikelyExploitable
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.Unknown, ConfidenceLevel.Low);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.LikelyExploitable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_L1L2ConfirmL3Conditional_ReturnsLikelyExploitable()
|
||||
{
|
||||
// L1=Reachable, L2=Resolved, L3=Conditional -> LikelyExploitable
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: true, GatingOutcome.Conditional, ConfidenceLevel.Medium);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.LikelyExploitable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_L1ReachableL2NotResolved_ReturnsUnreachable()
|
||||
{
|
||||
// L1=Reachable, L2=NotResolved (confirmed) -> Unreachable
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: false, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.Unreachable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_L1NotReachable_ReturnsUnreachable()
|
||||
{
|
||||
// L1=NotReachable (confirmed) -> Unreachable
|
||||
var layer1 = CreateLayer1(isReachable: false, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.Unreachable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_L3Blocked_ReturnsUnreachable()
|
||||
{
|
||||
// L1=Reachable, L2=Resolved, L3=Blocked (confirmed) -> Unreachable
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: true, GatingOutcome.Blocked, ConfidenceLevel.High);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.Unreachable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_L1ReachableL2LowConfidence_ReturnsPossiblyExploitable()
|
||||
{
|
||||
// L1=Reachable, L2=Unknown (low confidence) -> PossiblyExploitable
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: false, ConfidenceLevel.Low);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.Unknown, ConfidenceLevel.Low);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.PossiblyExploitable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_L1LowConfidenceNoData_ReturnsUnknown()
|
||||
{
|
||||
// L1=Unknown (low confidence, no paths) -> Unknown
|
||||
var layer1 = new ReachabilityLayer1
|
||||
{
|
||||
IsReachable = false,
|
||||
Confidence = ConfidenceLevel.Low,
|
||||
Paths = []
|
||||
};
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.Unknown);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Evaluate Tests
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_CreatesCompleteStack()
|
||||
{
|
||||
var symbol = CreateTestSymbol();
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
||||
|
||||
var stack = _evaluator.Evaluate("finding-123", symbol, layer1, layer2, layer3);
|
||||
|
||||
stack.Id.Should().NotBeNullOrEmpty();
|
||||
stack.FindingId.Should().Be("finding-123");
|
||||
stack.Symbol.Should().Be(symbol);
|
||||
stack.StaticCallGraph.Should().Be(layer1);
|
||||
stack.BinaryResolution.Should().Be(layer2);
|
||||
stack.RuntimeGating.Should().Be(layer3);
|
||||
stack.Verdict.Should().Be(ReachabilityVerdict.Exploitable);
|
||||
stack.AnalyzedAt.Should().BeCloseTo(DateTimeOffset.UtcNow, TimeSpan.FromSeconds(5));
|
||||
stack.Explanation.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_ExploitableVerdict_ExplanationContainsAllThreeLayers()
|
||||
{
|
||||
var symbol = CreateTestSymbol();
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
||||
|
||||
var stack = _evaluator.Evaluate("finding-123", symbol, layer1, layer2, layer3);
|
||||
|
||||
stack.Explanation.Should().Contain("Layer 1");
|
||||
stack.Explanation.Should().Contain("Layer 2");
|
||||
stack.Explanation.Should().Contain("Layer 3");
|
||||
stack.Explanation.Should().Contain("exploitable");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Evaluate_UnreachableVerdict_ExplanationMentionsBlocking()
|
||||
{
|
||||
var symbol = CreateTestSymbol();
|
||||
var layer1 = CreateLayer1(isReachable: false, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.High);
|
||||
|
||||
var stack = _evaluator.Evaluate("finding-123", symbol, layer1, layer2, layer3);
|
||||
|
||||
stack.Verdict.Should().Be(ReachabilityVerdict.Unreachable);
|
||||
stack.Explanation.Should().Contain("block");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Model Tests
|
||||
|
||||
[Fact]
|
||||
public void VulnerableSymbol_StoresAllProperties()
|
||||
{
|
||||
var symbol = new VulnerableSymbol(
|
||||
Name: "vulnerable_function",
|
||||
Library: "libvuln.so",
|
||||
Version: "2.0.0",
|
||||
VulnerabilityId: "CVE-2024-5678",
|
||||
Type: SymbolType.Function
|
||||
);
|
||||
|
||||
symbol.Name.Should().Be("vulnerable_function");
|
||||
symbol.Library.Should().Be("libvuln.so");
|
||||
symbol.Version.Should().Be("2.0.0");
|
||||
symbol.VulnerabilityId.Should().Be("CVE-2024-5678");
|
||||
symbol.Type.Should().Be(SymbolType.Function);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(SymbolType.Function)]
|
||||
[InlineData(SymbolType.Method)]
|
||||
[InlineData(SymbolType.JavaMethod)]
|
||||
[InlineData(SymbolType.JsFunction)]
|
||||
[InlineData(SymbolType.PyFunction)]
|
||||
[InlineData(SymbolType.GoFunction)]
|
||||
[InlineData(SymbolType.RustFunction)]
|
||||
public void SymbolType_AllValuesAreValid(SymbolType type)
|
||||
{
|
||||
var symbol = new VulnerableSymbol("test", null, null, "CVE-1234", type);
|
||||
symbol.Type.Should().Be(type);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(ReachabilityVerdict.Exploitable)]
|
||||
[InlineData(ReachabilityVerdict.LikelyExploitable)]
|
||||
[InlineData(ReachabilityVerdict.PossiblyExploitable)]
|
||||
[InlineData(ReachabilityVerdict.Unreachable)]
|
||||
[InlineData(ReachabilityVerdict.Unknown)]
|
||||
public void ReachabilityVerdict_AllValuesAreValid(ReachabilityVerdict verdict)
|
||||
{
|
||||
// Verify enum value is defined
|
||||
Enum.IsDefined(typeof(ReachabilityVerdict), verdict).Should().BeTrue();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(GatingOutcome.NotGated)]
|
||||
[InlineData(GatingOutcome.Blocked)]
|
||||
[InlineData(GatingOutcome.Conditional)]
|
||||
[InlineData(GatingOutcome.Unknown)]
|
||||
public void GatingOutcome_AllValuesAreValid(GatingOutcome outcome)
|
||||
{
|
||||
var layer3 = CreateLayer3(isGated: false, outcome, ConfidenceLevel.Medium);
|
||||
layer3.Outcome.Should().Be(outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GatingCondition_StoresAllProperties()
|
||||
{
|
||||
var condition = new GatingCondition(
|
||||
Type: GatingType.FeatureFlag,
|
||||
Description: "Feature flag check",
|
||||
ConfigKey: "feature.enabled",
|
||||
EnvVar: null,
|
||||
IsBlocking: true,
|
||||
Status: GatingStatus.Disabled
|
||||
);
|
||||
|
||||
condition.Type.Should().Be(GatingType.FeatureFlag);
|
||||
condition.Description.Should().Be("Feature flag check");
|
||||
condition.ConfigKey.Should().Be("feature.enabled");
|
||||
condition.IsBlocking.Should().BeTrue();
|
||||
condition.Status.Should().Be(GatingStatus.Disabled);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(GatingType.FeatureFlag)]
|
||||
[InlineData(GatingType.EnvironmentVariable)]
|
||||
[InlineData(GatingType.ConfigurationValue)]
|
||||
[InlineData(GatingType.CompileTimeConditional)]
|
||||
[InlineData(GatingType.PlatformCheck)]
|
||||
[InlineData(GatingType.CapabilityCheck)]
|
||||
[InlineData(GatingType.LicenseCheck)]
|
||||
[InlineData(GatingType.ExperimentFlag)]
|
||||
public void GatingType_AllValuesAreValid(GatingType type)
|
||||
{
|
||||
var condition = new GatingCondition(type, "test", null, null, false, GatingStatus.Unknown);
|
||||
condition.Type.Should().Be(type);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CallPath_WithSites_StoresCorrectly()
|
||||
{
|
||||
var entrypoint = new Entrypoint("Main", EntrypointType.Main, "Program.cs", "Application entry");
|
||||
var sites = new[]
|
||||
{
|
||||
new CallSite("Main", "Program", "Program.cs", 10, CallSiteType.Direct),
|
||||
new CallSite("ProcessData", "DataService", "DataService.cs", 45, CallSiteType.Virtual),
|
||||
new CallSite("vulnerable_function", null, "native.c", null, CallSiteType.Dynamic)
|
||||
};
|
||||
|
||||
var path = new CallPath
|
||||
{
|
||||
Sites = [.. sites],
|
||||
Entrypoint = entrypoint,
|
||||
Confidence = 0.85,
|
||||
HasConditionals = true
|
||||
};
|
||||
|
||||
path.Sites.Should().HaveCount(3);
|
||||
path.Entrypoint.Should().Be(entrypoint);
|
||||
path.Confidence.Should().Be(0.85);
|
||||
path.HasConditionals.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SymbolResolution_StoresDetails()
|
||||
{
|
||||
var resolution = new SymbolResolution(
|
||||
SymbolName: "EVP_DecryptUpdate",
|
||||
ResolvedLibrary: "/usr/lib/libcrypto.so.1.1",
|
||||
ResolvedVersion: "1.1.1k",
|
||||
SymbolVersion: "OPENSSL_1_1_0",
|
||||
Method: ResolutionMethod.DirectLink
|
||||
);
|
||||
|
||||
resolution.SymbolName.Should().Be("EVP_DecryptUpdate");
|
||||
resolution.ResolvedLibrary.Should().Be("/usr/lib/libcrypto.so.1.1");
|
||||
resolution.SymbolVersion.Should().Be("OPENSSL_1_1_0");
|
||||
resolution.Method.Should().Be(ResolutionMethod.DirectLink);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(ResolutionMethod.DirectLink)]
|
||||
[InlineData(ResolutionMethod.DynamicLoad)]
|
||||
[InlineData(ResolutionMethod.DelayLoad)]
|
||||
[InlineData(ResolutionMethod.WeakSymbol)]
|
||||
[InlineData(ResolutionMethod.Interposition)]
|
||||
public void ResolutionMethod_AllValuesAreValid(ResolutionMethod method)
|
||||
{
|
||||
var resolution = new SymbolResolution("sym", "lib", null, null, method);
|
||||
resolution.Method.Should().Be(method);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void LoaderRule_StoresProperties()
|
||||
{
|
||||
var rule = new LoaderRule(
|
||||
Type: LoaderRuleType.Rpath,
|
||||
Value: "/opt/myapp/lib",
|
||||
Source: "ELF binary"
|
||||
);
|
||||
|
||||
rule.Type.Should().Be(LoaderRuleType.Rpath);
|
||||
rule.Value.Should().Be("/opt/myapp/lib");
|
||||
rule.Source.Should().Be("ELF binary");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Edge Case Tests
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_L3BlockedButLowConfidence_DoesNotBlock()
|
||||
{
|
||||
// L3 blocked but low confidence should not definitively block
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.High);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.High);
|
||||
var layer3 = CreateLayer3(isGated: true, GatingOutcome.Blocked, ConfidenceLevel.Low);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
// With low confidence blocking, should still be exploitable since we can't trust the block
|
||||
verdict.Should().Be(ReachabilityVerdict.Exploitable);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeriveVerdict_AllLayersHighConfidence_ExploitableIsDefinitive()
|
||||
{
|
||||
var layer1 = CreateLayer1(isReachable: true, ConfidenceLevel.Verified);
|
||||
var layer2 = CreateLayer2(isResolved: true, ConfidenceLevel.Verified);
|
||||
var layer3 = CreateLayer3(isGated: false, GatingOutcome.NotGated, ConfidenceLevel.Verified);
|
||||
|
||||
var verdict = _evaluator.DeriveVerdict(layer1, layer2, layer3);
|
||||
|
||||
verdict.Should().Be(ReachabilityVerdict.Exploitable);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user