// ----------------------------------------------------------------------------- // ValidationHarnessServiceTests.cs // Sprint: SPRINT_20260121_034_BinaryIndex_golden_corpus_foundation // Task: GCF-003 - Implement validation harness skeleton // Description: Unit tests for ValidationHarnessService orchestration flow // ----------------------------------------------------------------------------- using System.Collections.Immutable; using FluentAssertions; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using StellaOps.BinaryIndex.GroundTruth.Abstractions; using Xunit; namespace StellaOps.BinaryIndex.GroundTruth.Reproducible.Tests; public sealed class ValidationHarnessServiceTests { private readonly ISecurityPairService _pairService; private readonly ValidationHarnessService _sut; public ValidationHarnessServiceTests() { _pairService = Substitute.For(); _sut = new ValidationHarnessService( _pairService, NullLogger.Instance); } #region Orchestration Flow Tests [Fact] public async Task RunAsync_EmptyPairs_ReturnsCompletedResult() { // Arrange var request = CreateValidationRequest([]); // Act var result = await _sut.RunAsync(request); // Assert result.Should().NotBeNull(); result.Status.State.Should().Be(ValidationState.Completed); result.PairResults.Should().BeEmpty(); result.Metrics.TotalPairs.Should().Be(0); result.Metrics.SuccessfulPairs.Should().Be(0); result.MarkdownReport.Should().NotBeNullOrEmpty(); } [Fact] public async Task RunAsync_SinglePair_ExecutesOrchestrationFlow() { // Arrange var pairRef = CreatePairReference("pair-001", "CVE-2024-1234", "libexample"); var securityPair = CreateSecurityPair(pairRef); _pairService.FindByIdAsync(pairRef.PairId, Arg.Any()) .Returns(securityPair); var request = CreateValidationRequest([pairRef]); // Act var result = await _sut.RunAsync(request); // Assert result.Should().NotBeNull(); result.Status.State.Should().Be(ValidationState.Completed); result.PairResults.Should().HaveCount(1); result.PairResults[0].PairId.Should().Be("pair-001"); result.PairResults[0].CveId.Should().Be("CVE-2024-1234"); result.PairResults[0].Success.Should().BeTrue(); result.RunId.Should().NotBeNullOrEmpty(); result.StartedAt.Should().BeBefore(result.CompletedAt); } [Fact] public async Task RunAsync_MultiplePairs_ProcessesAllPairs() { // Arrange var pairs = new[] { CreatePairReference("pair-001", "CVE-2024-1234", "libexample"), CreatePairReference("pair-002", "CVE-2024-5678", "libother"), CreatePairReference("pair-003", "CVE-2024-9999", "libthird") }; foreach (var pairRef in pairs) { var securityPair = CreateSecurityPair(pairRef); _pairService.FindByIdAsync(pairRef.PairId, Arg.Any()) .Returns(securityPair); } var request = CreateValidationRequest(pairs); // Act var result = await _sut.RunAsync(request); // Assert result.PairResults.Should().HaveCount(3); result.Metrics.TotalPairs.Should().Be(3); result.Metrics.SuccessfulPairs.Should().Be(3); } [Fact] public async Task RunAsync_PairNotFound_RecordsFailure() { // Arrange var pairRef = CreatePairReference("nonexistent", "CVE-2024-0000", "missing"); _pairService.FindByIdAsync(pairRef.PairId, Arg.Any()) .Returns((SecurityPair?)null); var request = CreateValidationRequest([pairRef]); // Act var result = await _sut.RunAsync(request); // Assert result.Status.State.Should().Be(ValidationState.Completed); result.PairResults.Should().HaveCount(1); result.PairResults[0].Success.Should().BeFalse(); result.PairResults[0].Error.Should().Contain("not found"); result.Metrics.FailedPairs.Should().Be(1); } [Fact] public async Task RunAsync_MixedResults_ContinuesOnFailure() { // Arrange var goodPair = CreatePairReference("pair-good", "CVE-2024-1111", "libgood"); var badPair = CreatePairReference("pair-bad", "CVE-2024-2222", "libbad"); _pairService.FindByIdAsync("pair-good", Arg.Any()) .Returns(CreateSecurityPair(goodPair)); _pairService.FindByIdAsync("pair-bad", Arg.Any()) .Returns((SecurityPair?)null); var request = new ValidationRunRequest { Pairs = [goodPair, badPair], Matcher = CreateMatcherConfig(), Metrics = CreateMetricsConfig(), ContinueOnFailure = true }; // Act var result = await _sut.RunAsync(request); // Assert result.Status.State.Should().Be(ValidationState.Completed); result.Metrics.SuccessfulPairs.Should().Be(1); result.Metrics.FailedPairs.Should().Be(1); } #endregion #region Status Tracking Tests [Fact] public async Task GetStatusAsync_UnknownRunId_ReturnsNull() { // Act var status = await _sut.GetStatusAsync("unknown-run-id"); // Assert status.Should().BeNull(); } [Fact] public async Task CancelAsync_UnknownRunId_ReturnsFalse() { // Act var cancelled = await _sut.CancelAsync("unknown-run-id"); // Assert cancelled.Should().BeFalse(); } #endregion #region Metrics Computation Tests [Fact] public async Task RunAsync_ComputesMetricsCorrectly() { // Arrange var pairRef = CreatePairReference("pair-001", "CVE-2024-1234", "libexample"); var securityPair = CreateSecurityPair(pairRef, changedFunctionCount: 2); _pairService.FindByIdAsync(pairRef.PairId, Arg.Any()) .Returns(securityPair); var request = CreateValidationRequest([pairRef]); // Act var result = await _sut.RunAsync(request); // Assert result.Metrics.Should().NotBeNull(); result.Metrics.TotalPairs.Should().Be(1); result.Metrics.SuccessfulPairs.Should().Be(1); // Note: FunctionMatchRate will be 0 because placeholder returns empty lists // This is expected for the skeleton implementation } #endregion #region Report Generation Tests [Fact] public async Task RunAsync_GeneratesMarkdownReport() { // Arrange var pairRef = CreatePairReference("pair-001", "CVE-2024-1234", "libexample"); var securityPair = CreateSecurityPair(pairRef); _pairService.FindByIdAsync(pairRef.PairId, Arg.Any()) .Returns(securityPair); var request = new ValidationRunRequest { Pairs = [pairRef], Matcher = CreateMatcherConfig(), Metrics = CreateMetricsConfig(), CorpusVersion = "v1.0.0" }; // Act var result = await _sut.RunAsync(request); // Assert result.MarkdownReport.Should().NotBeNullOrEmpty(); result.MarkdownReport.Should().Contain("# Validation Run Report"); result.MarkdownReport.Should().Contain("v1.0.0"); result.MarkdownReport.Should().Contain("Function Match Rate"); result.MarkdownReport.Should().Contain("False-Negative Rate"); result.MarkdownReport.Should().Contain("SBOM Hash Stability"); } [Fact] public async Task RunAsync_ReportContainsPairResults() { // Arrange var pairs = new[] { CreatePairReference("pair-001", "CVE-2024-1234", "libfirst"), CreatePairReference("pair-002", "CVE-2024-5678", "libsecond") }; foreach (var pairRef in pairs) { _pairService.FindByIdAsync(pairRef.PairId, Arg.Any()) .Returns(CreateSecurityPair(pairRef)); } var request = CreateValidationRequest(pairs); // Act var result = await _sut.RunAsync(request); // Assert result.MarkdownReport.Should().Contain("libfirst"); result.MarkdownReport.Should().Contain("libsecond"); result.MarkdownReport.Should().Contain("CVE-2024-1234"); result.MarkdownReport.Should().Contain("CVE-2024-5678"); } #endregion #region Timeout and Cancellation Tests [Fact] public async Task RunAsync_Cancellation_ReturnsCancelledOrFailedResult() { // Arrange var pairRef = CreatePairReference("pair-001", "CVE-2024-1234", "libexample"); var startedSemaphore = new SemaphoreSlim(0); // Make FindByIdAsync slow to allow cancellation _pairService.FindByIdAsync(pairRef.PairId, Arg.Any()) .Returns(async callInfo => { startedSemaphore.Release(); await Task.Delay(5000, callInfo.Arg()); return CreateSecurityPair(pairRef); }); var request = CreateValidationRequest([pairRef]); using var cts = new CancellationTokenSource(); // Act var runTask = _sut.RunAsync(request, cts.Token); // Wait for the operation to actually start await startedSemaphore.WaitAsync(TimeSpan.FromSeconds(5)); await cts.CancelAsync(); var result = await runTask; // Assert - may complete as cancelled or failed depending on timing result.Status.State.Should().BeOneOf( ValidationState.Cancelled, ValidationState.Failed, ValidationState.Completed); // May complete if cancellation is too slow // If completed, verify it handled the early return gracefully result.Should().NotBeNull(); } #endregion #region Configuration Tests [Fact] public async Task RunAsync_RespectsMaxParallelism() { // Arrange var pairs = Enumerable.Range(1, 10) .Select(i => CreatePairReference($"pair-{i:D3}", $"CVE-2024-{i:D4}", $"lib{i}")) .ToImmutableArray(); var concurrentCalls = 0; var maxConcurrentCalls = 0; var lockObj = new object(); foreach (var pairRef in pairs) { _pairService.FindByIdAsync(pairRef.PairId, Arg.Any()) .Returns(async _ => { lock (lockObj) { concurrentCalls++; maxConcurrentCalls = Math.Max(maxConcurrentCalls, concurrentCalls); } await Task.Delay(50); lock (lockObj) { concurrentCalls--; } return CreateSecurityPair(pairRef); }); } var request = new ValidationRunRequest { Pairs = pairs, Matcher = CreateMatcherConfig(), Metrics = CreateMetricsConfig(), MaxParallelism = 2 }; // Act await _sut.RunAsync(request); // Assert - max parallelism should not exceed configured value maxConcurrentCalls.Should().BeLessThanOrEqualTo(2); } #endregion #region Helper Methods private static ValidationRunRequest CreateValidationRequest( IEnumerable pairs) { return new ValidationRunRequest { Pairs = [.. pairs], Matcher = CreateMatcherConfig(), Metrics = CreateMetricsConfig() }; } private static MatcherConfiguration CreateMatcherConfig() { return new MatcherConfiguration { Algorithm = MatchingAlgorithm.Ensemble, MinimumSimilarity = 0.85, UseNameMatching = true, UseStructuralMatching = true, UseSemanticMatching = true }; } private static MetricsConfiguration CreateMetricsConfig() { return new MetricsConfiguration { ComputeMatchRate = true, ComputeFalseNegativeRate = true, VerifySbomStability = true, SbomStabilityRuns = 3, GenerateMismatchBuckets = true }; } private static SecurityPairReference CreatePairReference( string pairId, string cveId, string packageName) { return new SecurityPairReference { PairId = pairId, CveId = cveId, PackageName = packageName, VulnerableVersion = "1.0.0", PatchedVersion = "1.0.1" }; } private static SecurityPair CreateSecurityPair( SecurityPairReference pairRef, int changedFunctionCount = 1) { var changedFunctions = Enumerable.Range(1, changedFunctionCount) .Select(i => new ChangedFunction( $"vuln_function_{i}", VulnerableSize: 100 + i * 10, PatchedSize: 120 + i * 10, SizeDelta: 20, ChangeType.Modified, "Security fix")) .ToImmutableArray(); return new SecurityPair { PairId = pairRef.PairId, CveId = pairRef.CveId, PackageName = pairRef.PackageName, VulnerableVersion = pairRef.VulnerableVersion, PatchedVersion = pairRef.PatchedVersion, Distro = "debian", VulnerableObservationId = $"obs-vuln-{pairRef.PairId}", VulnerableDebugId = $"dbg-vuln-{pairRef.PairId}", PatchedObservationId = $"obs-patch-{pairRef.PairId}", PatchedDebugId = $"dbg-patch-{pairRef.PairId}", AffectedFunctions = [new AffectedFunction( "vulnerable_func", VulnerableAddress: 0x1000, PatchedAddress: 0x1000, AffectedFunctionType.Vulnerable, "Main vulnerability")], ChangedFunctions = changedFunctions, CreatedAt = DateTimeOffset.UtcNow }; } #endregion }