528 lines
20 KiB
C#
528 lines
20 KiB
C#
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// Copyright (c) 2025-2026 StellaOps
|
|
// Sprint: SPRINT_20260109_009_005_BE_vex_decision_integration
|
|
// Task: Integration tests for VEX decision with hybrid reachability
|
|
|
|
using System.Collections.Immutable;
|
|
using FluentAssertions;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using MsOptions = Microsoft.Extensions.Options;
|
|
using Moq;
|
|
using StellaOps.Determinism;
|
|
using StellaOps.Policy.Engine.Gates;
|
|
using StellaOps.Policy.Engine.ReachabilityFacts;
|
|
using StellaOps.Policy.Engine.Vex;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Policy.Engine.Tests.Vex;
|
|
|
|
/// <summary>
|
|
/// Integration tests for VEX decision emission with hybrid reachability evidence.
|
|
/// Tests the full pipeline from reachability facts to VEX document generation.
|
|
/// </summary>
|
|
[Trait("Category", "Integration")]
|
|
[Trait("Sprint", "009_005")]
|
|
public sealed class VexDecisionReachabilityIntegrationTests
|
|
{
|
|
private const string TestTenantId = "integration-test-tenant";
|
|
private const string TestAuthor = "vex-emitter@stellaops.test";
|
|
|
|
#region End-to-End Pipeline Tests
|
|
|
|
[Fact(DisplayName = "Pipeline emits VEX for multiple findings with varying reachability states")]
|
|
public async Task Pipeline_EmitsVex_ForMultipleFindingsWithVaryingStates()
|
|
{
|
|
// Arrange: Create findings with different reachability states
|
|
var findings = new[]
|
|
{
|
|
new VexFindingInput { VulnId = "CVE-2024-0001", Purl = "pkg:npm/lodash@4.17.20" },
|
|
new VexFindingInput { VulnId = "CVE-2024-0002", Purl = "pkg:maven/log4j/log4j-core@2.14.1" },
|
|
new VexFindingInput { VulnId = "CVE-2024-0003", Purl = "pkg:pypi/requests@2.25.0" }
|
|
};
|
|
|
|
var facts = new Dictionary<ReachabilityFactKey, ReachabilityFact>
|
|
{
|
|
[new(TestTenantId, "pkg:npm/lodash@4.17.20", "CVE-2024-0001")] = CreateFact(
|
|
TestTenantId, "pkg:npm/lodash@4.17.20", "CVE-2024-0001",
|
|
ReachabilityState.Unreachable,
|
|
hasRuntime: true,
|
|
confidence: 0.95m),
|
|
[new(TestTenantId, "pkg:maven/log4j/log4j-core@2.14.1", "CVE-2024-0002")] = CreateFact(
|
|
TestTenantId, "pkg:maven/log4j/log4j-core@2.14.1", "CVE-2024-0002",
|
|
ReachabilityState.Reachable,
|
|
hasRuntime: true,
|
|
confidence: 0.99m),
|
|
[new(TestTenantId, "pkg:pypi/requests@2.25.0", "CVE-2024-0003")] = CreateFact(
|
|
TestTenantId, "pkg:pypi/requests@2.25.0", "CVE-2024-0003",
|
|
ReachabilityState.Unknown,
|
|
hasRuntime: false,
|
|
confidence: 0.0m)
|
|
};
|
|
|
|
var factsService = CreateMockFactsService(facts);
|
|
var gateEvaluator = CreateMockGateEvaluator(PolicyGateDecisionType.Allow);
|
|
var emitter = CreateEmitter(factsService, gateEvaluator);
|
|
|
|
var request = new VexDecisionEmitRequest
|
|
{
|
|
TenantId = TestTenantId,
|
|
Author = TestAuthor,
|
|
Findings = findings
|
|
};
|
|
|
|
// Act
|
|
var result = await emitter.EmitAsync(request);
|
|
|
|
// Assert
|
|
result.Document.Should().NotBeNull();
|
|
result.Document.Statements.Should().HaveCount(3);
|
|
result.Blocked.Should().BeEmpty();
|
|
|
|
// Verify unreachable -> not_affected
|
|
var lodashStatement = result.Document.Statements.Single(s => s.Vulnerability.Id == "CVE-2024-0001");
|
|
lodashStatement.Status.Should().Be("not_affected");
|
|
lodashStatement.Justification.Should().Be(VexJustification.VulnerableCodeNotInExecutePath);
|
|
|
|
// Verify reachable -> affected
|
|
var log4jStatement = result.Document.Statements.Single(s => s.Vulnerability.Id == "CVE-2024-0002");
|
|
log4jStatement.Status.Should().Be("affected");
|
|
log4jStatement.Justification.Should().BeNull();
|
|
|
|
// Verify unknown -> under_investigation
|
|
var requestsStatement = result.Document.Statements.Single(s => s.Vulnerability.Id == "CVE-2024-0003");
|
|
requestsStatement.Status.Should().Be("under_investigation");
|
|
}
|
|
|
|
[Fact(DisplayName = "Pipeline preserves evidence hash in VEX metadata")]
|
|
public async Task Pipeline_PreservesEvidenceHash_InVexMetadata()
|
|
{
|
|
// Arrange
|
|
const string expectedHash = "sha256:abc123def456";
|
|
var facts = new Dictionary<ReachabilityFactKey, ReachabilityFact>
|
|
{
|
|
[new(TestTenantId, "pkg:npm/vulnerable@1.0.0", "CVE-2024-1000")] = CreateFact(
|
|
TestTenantId, "pkg:npm/vulnerable@1.0.0", "CVE-2024-1000",
|
|
ReachabilityState.Unreachable,
|
|
hasRuntime: true,
|
|
confidence: 0.92m,
|
|
evidenceHash: expectedHash)
|
|
};
|
|
|
|
var factsService = CreateMockFactsService(facts);
|
|
var gateEvaluator = CreateMockGateEvaluator(PolicyGateDecisionType.Allow);
|
|
var emitter = CreateEmitter(factsService, gateEvaluator);
|
|
|
|
var request = new VexDecisionEmitRequest
|
|
{
|
|
TenantId = TestTenantId,
|
|
Author = TestAuthor,
|
|
Findings = new[] { new VexFindingInput { VulnId = "CVE-2024-1000", Purl = "pkg:npm/vulnerable@1.0.0" } }
|
|
};
|
|
|
|
// Act
|
|
var result = await emitter.EmitAsync(request);
|
|
|
|
// Assert
|
|
result.Document.Should().NotBeNull();
|
|
var statement = result.Document.Statements.Should().ContainSingle().Subject;
|
|
statement.Evidence.Should().NotBeNull();
|
|
statement.Evidence!.GraphHash.Should().Be(expectedHash);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Policy Gate Integration Tests
|
|
|
|
[Fact(DisplayName = "Policy gate blocks emission for high-risk findings")]
|
|
public async Task PolicyGate_BlocksEmission_ForHighRiskFindings()
|
|
{
|
|
// Arrange
|
|
var facts = new Dictionary<ReachabilityFactKey, ReachabilityFact>
|
|
{
|
|
[new(TestTenantId, "pkg:npm/critical@1.0.0", "CVE-2024-CRITICAL")] = CreateFact(
|
|
TestTenantId, "pkg:npm/critical@1.0.0", "CVE-2024-CRITICAL",
|
|
ReachabilityState.Reachable,
|
|
hasRuntime: true,
|
|
confidence: 0.99m)
|
|
};
|
|
|
|
var factsService = CreateMockFactsService(facts);
|
|
var gateEvaluator = CreateMockGateEvaluator(
|
|
PolicyGateDecisionType.Block,
|
|
blockedBy: "SecurityReviewGate",
|
|
reason: "Requires security review for critical CVEs");
|
|
var emitter = CreateEmitter(factsService, gateEvaluator);
|
|
|
|
var request = new VexDecisionEmitRequest
|
|
{
|
|
TenantId = TestTenantId,
|
|
Author = TestAuthor,
|
|
Findings = new[] { new VexFindingInput { VulnId = "CVE-2024-CRITICAL", Purl = "pkg:npm/critical@1.0.0" } }
|
|
};
|
|
|
|
// Act
|
|
var result = await emitter.EmitAsync(request);
|
|
|
|
// Assert
|
|
result.Blocked.Should().ContainSingle();
|
|
result.Blocked[0].VulnId.Should().Be("CVE-2024-CRITICAL");
|
|
result.Blocked[0].Reason.Should().Contain("security review");
|
|
}
|
|
|
|
[Fact(DisplayName = "Policy gate warns but allows emission when configured")]
|
|
public async Task PolicyGate_WarnsButAllows_WhenConfigured()
|
|
{
|
|
// Arrange
|
|
var facts = new Dictionary<ReachabilityFactKey, ReachabilityFact>
|
|
{
|
|
[new(TestTenantId, "pkg:npm/medium@1.0.0", "CVE-2024-MEDIUM")] = CreateFact(
|
|
TestTenantId, "pkg:npm/medium@1.0.0", "CVE-2024-MEDIUM",
|
|
ReachabilityState.Unreachable,
|
|
hasRuntime: true,
|
|
confidence: 0.85m)
|
|
};
|
|
|
|
var factsService = CreateMockFactsService(facts);
|
|
var gateEvaluator = CreateMockGateEvaluator(
|
|
PolicyGateDecisionType.Warn,
|
|
advisory: "Confidence below threshold");
|
|
var emitter = CreateEmitter(factsService, gateEvaluator);
|
|
|
|
var request = new VexDecisionEmitRequest
|
|
{
|
|
TenantId = TestTenantId,
|
|
Author = TestAuthor,
|
|
Findings = new[] { new VexFindingInput { VulnId = "CVE-2024-MEDIUM", Purl = "pkg:npm/medium@1.0.0" } }
|
|
};
|
|
|
|
// Act
|
|
var result = await emitter.EmitAsync(request);
|
|
|
|
// Assert
|
|
result.Document.Should().NotBeNull();
|
|
result.Document.Statements.Should().ContainSingle();
|
|
result.Blocked.Should().BeEmpty();
|
|
// Warnings should be logged but emission continues
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Lattice State Integration Tests
|
|
|
|
[Theory(DisplayName = "All lattice states map to correct VEX status")]
|
|
[InlineData("U", "under_investigation")]
|
|
[InlineData("SR", "under_investigation")] // Static-only needs runtime confirmation
|
|
[InlineData("SU", "not_affected")]
|
|
[InlineData("RO", "affected")] // Runtime observed = definitely reachable
|
|
[InlineData("RU", "not_affected")]
|
|
[InlineData("CR", "affected")]
|
|
[InlineData("CU", "not_affected")]
|
|
// Note: "X" (Contested) maps to Unknown state and under_investigation status
|
|
public async Task LatticeState_MapsToCorrectVexStatus(string latticeState, string expectedStatus)
|
|
{
|
|
// Arrange
|
|
var state = latticeState switch
|
|
{
|
|
"U" => ReachabilityState.Unknown,
|
|
"SR" or "RO" or "CR" => ReachabilityState.Reachable,
|
|
"SU" or "RU" or "CU" => ReachabilityState.Unreachable,
|
|
_ => ReachabilityState.Unknown
|
|
};
|
|
|
|
var hasRuntime = latticeState is "RO" or "RU" or "CR" or "CU";
|
|
var confidence = latticeState switch
|
|
{
|
|
"CR" or "CU" => 0.95m,
|
|
"RO" or "RU" => 0.85m,
|
|
"SR" or "SU" => 0.70m,
|
|
_ => 0.0m
|
|
};
|
|
|
|
var facts = new Dictionary<ReachabilityFactKey, ReachabilityFact>
|
|
{
|
|
[new(TestTenantId, "pkg:test/lib@1.0.0", "CVE-TEST")] = CreateFact(
|
|
TestTenantId, "pkg:test/lib@1.0.0", "CVE-TEST",
|
|
state,
|
|
hasRuntime: hasRuntime,
|
|
confidence: confidence)
|
|
};
|
|
|
|
var factsService = CreateMockFactsService(facts);
|
|
var gateEvaluator = CreateMockGateEvaluator(PolicyGateDecisionType.Allow);
|
|
var emitter = CreateEmitter(factsService, gateEvaluator);
|
|
|
|
var request = new VexDecisionEmitRequest
|
|
{
|
|
TenantId = TestTenantId,
|
|
Author = TestAuthor,
|
|
Findings = new[] { new VexFindingInput { VulnId = "CVE-TEST", Purl = "pkg:test/lib@1.0.0" } }
|
|
};
|
|
|
|
// Act
|
|
var result = await emitter.EmitAsync(request);
|
|
|
|
// Assert
|
|
result.Document.Should().NotBeNull();
|
|
var statement = result.Document.Statements.Should().ContainSingle().Subject;
|
|
statement.Status.Should().Be(expectedStatus);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Override Integration Tests
|
|
|
|
[Fact(DisplayName = "Manual override takes precedence over reachability")]
|
|
public async Task ManualOverride_TakesPrecedence_OverReachability()
|
|
{
|
|
// Arrange: Reachable CVE with manual override to not_affected
|
|
var facts = new Dictionary<ReachabilityFactKey, ReachabilityFact>
|
|
{
|
|
[new(TestTenantId, "pkg:npm/overridden@1.0.0", "CVE-2024-OVERRIDE")] = CreateFact(
|
|
TestTenantId, "pkg:npm/overridden@1.0.0", "CVE-2024-OVERRIDE",
|
|
ReachabilityState.Reachable,
|
|
hasRuntime: true,
|
|
confidence: 0.99m)
|
|
};
|
|
|
|
var factsService = CreateMockFactsService(facts);
|
|
var gateEvaluator = CreateMockGateEvaluator(PolicyGateDecisionType.Allow);
|
|
var emitter = CreateEmitter(factsService, gateEvaluator);
|
|
|
|
var request = new VexDecisionEmitRequest
|
|
{
|
|
TenantId = TestTenantId,
|
|
Author = TestAuthor,
|
|
Findings = new[]
|
|
{
|
|
new VexFindingInput
|
|
{
|
|
VulnId = "CVE-2024-OVERRIDE",
|
|
Purl = "pkg:npm/overridden@1.0.0",
|
|
OverrideStatus = "not_affected",
|
|
OverrideJustification = "Vulnerable path protected by WAF rules"
|
|
}
|
|
}
|
|
};
|
|
|
|
// Act
|
|
var result = await emitter.EmitAsync(request);
|
|
|
|
// Assert
|
|
result.Document.Should().NotBeNull();
|
|
var statement = result.Document.Statements.Should().ContainSingle().Subject;
|
|
statement.Status.Should().Be("not_affected");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Determinism Tests
|
|
|
|
[Fact(DisplayName = "Same inputs produce identical VEX documents")]
|
|
public async Task Determinism_SameInputs_ProduceIdenticalDocuments()
|
|
{
|
|
// Arrange
|
|
var facts = new Dictionary<ReachabilityFactKey, ReachabilityFact>
|
|
{
|
|
[new(TestTenantId, "pkg:npm/deterministic@1.0.0", "CVE-2024-DET")] = CreateFact(
|
|
TestTenantId, "pkg:npm/deterministic@1.0.0", "CVE-2024-DET",
|
|
ReachabilityState.Unreachable,
|
|
hasRuntime: true,
|
|
confidence: 0.95m)
|
|
};
|
|
|
|
var factsService = CreateMockFactsService(facts);
|
|
var gateEvaluator = CreateMockGateEvaluator(PolicyGateDecisionType.Allow);
|
|
|
|
// Use fixed time for determinism
|
|
var fixedTime = new DateTimeOffset(2026, 1, 10, 12, 0, 0, TimeSpan.Zero);
|
|
var timeProvider = new FakeTimeProvider(fixedTime);
|
|
|
|
var emitter = CreateEmitter(factsService, gateEvaluator, timeProvider);
|
|
|
|
var request = new VexDecisionEmitRequest
|
|
{
|
|
TenantId = TestTenantId,
|
|
Author = TestAuthor,
|
|
Findings = new[] { new VexFindingInput { VulnId = "CVE-2024-DET", Purl = "pkg:npm/deterministic@1.0.0" } }
|
|
};
|
|
|
|
// Act
|
|
var result1 = await emitter.EmitAsync(request);
|
|
var result2 = await emitter.EmitAsync(request);
|
|
|
|
// Assert
|
|
result1.Document.Should().NotBeNull();
|
|
result2.Document.Should().NotBeNull();
|
|
|
|
// Both documents should have identical content
|
|
result1.Document.Statements.Should().HaveCount(result2.Document.Statements.Length);
|
|
|
|
var stmt1 = result1.Document.Statements[0];
|
|
var stmt2 = result2.Document.Statements[0];
|
|
|
|
stmt1.Status.Should().Be(stmt2.Status);
|
|
stmt1.Justification.Should().Be(stmt2.Justification);
|
|
stmt1.Vulnerability.Id.Should().Be(stmt2.Vulnerability.Id);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Helper Methods
|
|
|
|
private static ReachabilityFact CreateFact(
|
|
string tenantId,
|
|
string componentPurl,
|
|
string advisoryId,
|
|
ReachabilityState state,
|
|
bool hasRuntime,
|
|
decimal confidence,
|
|
string? evidenceHash = null)
|
|
{
|
|
return new ReachabilityFact
|
|
{
|
|
Id = Guid.NewGuid().ToString(),
|
|
TenantId = tenantId,
|
|
ComponentPurl = componentPurl,
|
|
AdvisoryId = advisoryId,
|
|
State = state,
|
|
Confidence = confidence,
|
|
Score = state == ReachabilityState.Reachable ? 1.0m : 0.0m,
|
|
HasRuntimeEvidence = hasRuntime,
|
|
Source = "test-source",
|
|
Method = hasRuntime ? AnalysisMethod.Hybrid : AnalysisMethod.Static,
|
|
EvidenceHash = evidenceHash,
|
|
ComputedAt = DateTimeOffset.UtcNow
|
|
};
|
|
}
|
|
|
|
private static ReachabilityFactsJoiningService CreateMockFactsService(
|
|
Dictionary<ReachabilityFactKey, ReachabilityFact> facts)
|
|
{
|
|
var storeMock = new Mock<IReachabilityFactsStore>();
|
|
var cacheMock = new Mock<IReachabilityFactsOverlayCache>();
|
|
var logger = NullLogger<ReachabilityFactsJoiningService>.Instance;
|
|
|
|
// Setup cache to return misses initially, forcing store lookup
|
|
cacheMock
|
|
.Setup(c => c.GetBatchAsync(
|
|
It.IsAny<IReadOnlyList<ReachabilityFactKey>>(),
|
|
It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync((IReadOnlyList<ReachabilityFactKey> keys, CancellationToken _) =>
|
|
{
|
|
return new ReachabilityFactsBatch
|
|
{
|
|
Found = new Dictionary<ReachabilityFactKey, ReachabilityFact>(),
|
|
NotFound = keys.ToList(),
|
|
CacheHits = 0,
|
|
CacheMisses = keys.Count
|
|
};
|
|
});
|
|
|
|
// Setup store to return facts
|
|
storeMock
|
|
.Setup(s => s.GetBatchAsync(
|
|
It.IsAny<IReadOnlyList<ReachabilityFactKey>>(),
|
|
It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync((IReadOnlyList<ReachabilityFactKey> keys, CancellationToken _) =>
|
|
{
|
|
var found = new Dictionary<ReachabilityFactKey, ReachabilityFact>();
|
|
foreach (var key in keys)
|
|
{
|
|
if (facts.TryGetValue(key, out var fact))
|
|
{
|
|
found[key] = fact;
|
|
}
|
|
}
|
|
|
|
return found;
|
|
});
|
|
|
|
// Setup cache set (no-op)
|
|
cacheMock
|
|
.Setup(c => c.SetBatchAsync(
|
|
It.IsAny<IReadOnlyDictionary<ReachabilityFactKey, ReachabilityFact>>(),
|
|
It.IsAny<CancellationToken>()))
|
|
.Returns(Task.CompletedTask);
|
|
|
|
return new ReachabilityFactsJoiningService(
|
|
storeMock.Object,
|
|
cacheMock.Object,
|
|
logger,
|
|
TimeProvider.System);
|
|
}
|
|
|
|
private static IPolicyGateEvaluator CreateMockGateEvaluator(
|
|
PolicyGateDecisionType decision,
|
|
string? blockedBy = null,
|
|
string? reason = null,
|
|
string? advisory = null)
|
|
{
|
|
var mock = new Mock<IPolicyGateEvaluator>();
|
|
mock.Setup(e => e.EvaluateAsync(It.IsAny<PolicyGateRequest>(), It.IsAny<CancellationToken>()))
|
|
.ReturnsAsync((PolicyGateRequest req, CancellationToken _) => new PolicyGateDecision
|
|
{
|
|
GateId = Guid.NewGuid().ToString(),
|
|
RequestedStatus = req.RequestedStatus,
|
|
Subject = new PolicyGateSubject
|
|
{
|
|
VulnId = req.VulnId,
|
|
Purl = req.Purl
|
|
},
|
|
Evidence = new PolicyGateEvidence
|
|
{
|
|
LatticeState = req.LatticeState,
|
|
Confidence = req.Confidence,
|
|
HasRuntimeEvidence = req.HasRuntimeEvidence
|
|
},
|
|
Gates = ImmutableArray<PolicyGateResult>.Empty,
|
|
Decision = decision,
|
|
BlockedBy = blockedBy,
|
|
BlockReason = reason,
|
|
Advisory = advisory,
|
|
DecidedAt = DateTimeOffset.UtcNow
|
|
});
|
|
return mock.Object;
|
|
}
|
|
|
|
private static VexDecisionEmitter CreateEmitter(
|
|
ReachabilityFactsJoiningService factsService,
|
|
IPolicyGateEvaluator gateEvaluator,
|
|
TimeProvider? timeProvider = null)
|
|
{
|
|
var options = MsOptions.Options.Create(new VexDecisionEmitterOptions
|
|
{
|
|
MinConfidenceForNotAffected = 0.7,
|
|
RequireRuntimeForNotAffected = false
|
|
});
|
|
|
|
return new VexDecisionEmitter(
|
|
factsService,
|
|
gateEvaluator,
|
|
new OptionsMonitorWrapper<VexDecisionEmitterOptions>(options.Value),
|
|
timeProvider ?? TimeProvider.System,
|
|
SystemGuidProvider.Instance,
|
|
NullLogger<VexDecisionEmitter>.Instance);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Test Helpers
|
|
|
|
private sealed class OptionsMonitorWrapper<T> : MsOptions.IOptionsMonitor<T>
|
|
{
|
|
public OptionsMonitorWrapper(T value) => CurrentValue = value;
|
|
public T CurrentValue { get; }
|
|
public T Get(string? name) => CurrentValue;
|
|
public IDisposable? OnChange(Action<T, string?> listener) => null;
|
|
}
|
|
|
|
private sealed class FakeTimeProvider : TimeProvider
|
|
{
|
|
private readonly DateTimeOffset _fixedTime;
|
|
public FakeTimeProvider(DateTimeOffset fixedTime) => _fixedTime = fixedTime;
|
|
public override DateTimeOffset GetUtcNow() => _fixedTime;
|
|
}
|
|
|
|
#endregion
|
|
}
|