454 lines
14 KiB
C#
454 lines
14 KiB
C#
// -----------------------------------------------------------------------------
|
|
// ValidationHarnessServiceTests.cs
|
|
// Sprint: SPRINT_20260121_034_BinaryIndex_golden_corpus_foundation
|
|
// Task: GCF-003 - Implement validation harness skeleton
|
|
// Description: Unit tests for ValidationHarnessService orchestration flow
|
|
// -----------------------------------------------------------------------------
|
|
|
|
using System.Collections.Immutable;
|
|
using FluentAssertions;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using NSubstitute;
|
|
using StellaOps.BinaryIndex.GroundTruth.Abstractions;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.BinaryIndex.GroundTruth.Reproducible.Tests;
|
|
|
|
public sealed class ValidationHarnessServiceTests
|
|
{
|
|
private readonly ISecurityPairService _pairService;
|
|
private readonly ValidationHarnessService _sut;
|
|
|
|
public ValidationHarnessServiceTests()
|
|
{
|
|
_pairService = Substitute.For<ISecurityPairService>();
|
|
_sut = new ValidationHarnessService(
|
|
_pairService,
|
|
NullLogger<ValidationHarnessService>.Instance);
|
|
}
|
|
|
|
#region Orchestration Flow Tests
|
|
|
|
[Fact]
|
|
public async Task RunAsync_EmptyPairs_ReturnsCompletedResult()
|
|
{
|
|
// Arrange
|
|
var request = CreateValidationRequest([]);
|
|
|
|
// Act
|
|
var result = await _sut.RunAsync(request);
|
|
|
|
// Assert
|
|
result.Should().NotBeNull();
|
|
result.Status.State.Should().Be(ValidationState.Completed);
|
|
result.PairResults.Should().BeEmpty();
|
|
result.Metrics.TotalPairs.Should().Be(0);
|
|
result.Metrics.SuccessfulPairs.Should().Be(0);
|
|
result.MarkdownReport.Should().NotBeNullOrEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RunAsync_SinglePair_ExecutesOrchestrationFlow()
|
|
{
|
|
// Arrange
|
|
var pairRef = CreatePairReference("pair-001", "CVE-2024-1234", "libexample");
|
|
var securityPair = CreateSecurityPair(pairRef);
|
|
|
|
_pairService.FindByIdAsync(pairRef.PairId, Arg.Any<CancellationToken>())
|
|
.Returns(securityPair);
|
|
|
|
var request = CreateValidationRequest([pairRef]);
|
|
|
|
// Act
|
|
var result = await _sut.RunAsync(request);
|
|
|
|
// Assert
|
|
result.Should().NotBeNull();
|
|
result.Status.State.Should().Be(ValidationState.Completed);
|
|
result.PairResults.Should().HaveCount(1);
|
|
result.PairResults[0].PairId.Should().Be("pair-001");
|
|
result.PairResults[0].CveId.Should().Be("CVE-2024-1234");
|
|
result.PairResults[0].Success.Should().BeTrue();
|
|
result.RunId.Should().NotBeNullOrEmpty();
|
|
result.StartedAt.Should().BeBefore(result.CompletedAt);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RunAsync_MultiplePairs_ProcessesAllPairs()
|
|
{
|
|
// Arrange
|
|
var pairs = new[]
|
|
{
|
|
CreatePairReference("pair-001", "CVE-2024-1234", "libexample"),
|
|
CreatePairReference("pair-002", "CVE-2024-5678", "libother"),
|
|
CreatePairReference("pair-003", "CVE-2024-9999", "libthird")
|
|
};
|
|
|
|
foreach (var pairRef in pairs)
|
|
{
|
|
var securityPair = CreateSecurityPair(pairRef);
|
|
_pairService.FindByIdAsync(pairRef.PairId, Arg.Any<CancellationToken>())
|
|
.Returns(securityPair);
|
|
}
|
|
|
|
var request = CreateValidationRequest(pairs);
|
|
|
|
// Act
|
|
var result = await _sut.RunAsync(request);
|
|
|
|
// Assert
|
|
result.PairResults.Should().HaveCount(3);
|
|
result.Metrics.TotalPairs.Should().Be(3);
|
|
result.Metrics.SuccessfulPairs.Should().Be(3);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RunAsync_PairNotFound_RecordsFailure()
|
|
{
|
|
// Arrange
|
|
var pairRef = CreatePairReference("nonexistent", "CVE-2024-0000", "missing");
|
|
|
|
_pairService.FindByIdAsync(pairRef.PairId, Arg.Any<CancellationToken>())
|
|
.Returns((SecurityPair?)null);
|
|
|
|
var request = CreateValidationRequest([pairRef]);
|
|
|
|
// Act
|
|
var result = await _sut.RunAsync(request);
|
|
|
|
// Assert
|
|
result.Status.State.Should().Be(ValidationState.Completed);
|
|
result.PairResults.Should().HaveCount(1);
|
|
result.PairResults[0].Success.Should().BeFalse();
|
|
result.PairResults[0].Error.Should().Contain("not found");
|
|
result.Metrics.FailedPairs.Should().Be(1);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RunAsync_MixedResults_ContinuesOnFailure()
|
|
{
|
|
// Arrange
|
|
var goodPair = CreatePairReference("pair-good", "CVE-2024-1111", "libgood");
|
|
var badPair = CreatePairReference("pair-bad", "CVE-2024-2222", "libbad");
|
|
|
|
_pairService.FindByIdAsync("pair-good", Arg.Any<CancellationToken>())
|
|
.Returns(CreateSecurityPair(goodPair));
|
|
_pairService.FindByIdAsync("pair-bad", Arg.Any<CancellationToken>())
|
|
.Returns((SecurityPair?)null);
|
|
|
|
var request = new ValidationRunRequest
|
|
{
|
|
Pairs = [goodPair, badPair],
|
|
Matcher = CreateMatcherConfig(),
|
|
Metrics = CreateMetricsConfig(),
|
|
ContinueOnFailure = true
|
|
};
|
|
|
|
// Act
|
|
var result = await _sut.RunAsync(request);
|
|
|
|
// Assert
|
|
result.Status.State.Should().Be(ValidationState.Completed);
|
|
result.Metrics.SuccessfulPairs.Should().Be(1);
|
|
result.Metrics.FailedPairs.Should().Be(1);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Status Tracking Tests
|
|
|
|
[Fact]
|
|
public async Task GetStatusAsync_UnknownRunId_ReturnsNull()
|
|
{
|
|
// Act
|
|
var status = await _sut.GetStatusAsync("unknown-run-id");
|
|
|
|
// Assert
|
|
status.Should().BeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CancelAsync_UnknownRunId_ReturnsFalse()
|
|
{
|
|
// Act
|
|
var cancelled = await _sut.CancelAsync("unknown-run-id");
|
|
|
|
// Assert
|
|
cancelled.Should().BeFalse();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Metrics Computation Tests
|
|
|
|
[Fact]
|
|
public async Task RunAsync_ComputesMetricsCorrectly()
|
|
{
|
|
// Arrange
|
|
var pairRef = CreatePairReference("pair-001", "CVE-2024-1234", "libexample");
|
|
var securityPair = CreateSecurityPair(pairRef, changedFunctionCount: 2);
|
|
|
|
_pairService.FindByIdAsync(pairRef.PairId, Arg.Any<CancellationToken>())
|
|
.Returns(securityPair);
|
|
|
|
var request = CreateValidationRequest([pairRef]);
|
|
|
|
// Act
|
|
var result = await _sut.RunAsync(request);
|
|
|
|
// Assert
|
|
result.Metrics.Should().NotBeNull();
|
|
result.Metrics.TotalPairs.Should().Be(1);
|
|
result.Metrics.SuccessfulPairs.Should().Be(1);
|
|
// Note: FunctionMatchRate will be 0 because placeholder returns empty lists
|
|
// This is expected for the skeleton implementation
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Report Generation Tests
|
|
|
|
[Fact]
|
|
public async Task RunAsync_GeneratesMarkdownReport()
|
|
{
|
|
// Arrange
|
|
var pairRef = CreatePairReference("pair-001", "CVE-2024-1234", "libexample");
|
|
var securityPair = CreateSecurityPair(pairRef);
|
|
|
|
_pairService.FindByIdAsync(pairRef.PairId, Arg.Any<CancellationToken>())
|
|
.Returns(securityPair);
|
|
|
|
var request = new ValidationRunRequest
|
|
{
|
|
Pairs = [pairRef],
|
|
Matcher = CreateMatcherConfig(),
|
|
Metrics = CreateMetricsConfig(),
|
|
CorpusVersion = "v1.0.0"
|
|
};
|
|
|
|
// Act
|
|
var result = await _sut.RunAsync(request);
|
|
|
|
// Assert
|
|
result.MarkdownReport.Should().NotBeNullOrEmpty();
|
|
result.MarkdownReport.Should().Contain("# Validation Run Report");
|
|
result.MarkdownReport.Should().Contain("v1.0.0");
|
|
result.MarkdownReport.Should().Contain("Function Match Rate");
|
|
result.MarkdownReport.Should().Contain("False-Negative Rate");
|
|
result.MarkdownReport.Should().Contain("SBOM Hash Stability");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RunAsync_ReportContainsPairResults()
|
|
{
|
|
// Arrange
|
|
var pairs = new[]
|
|
{
|
|
CreatePairReference("pair-001", "CVE-2024-1234", "libfirst"),
|
|
CreatePairReference("pair-002", "CVE-2024-5678", "libsecond")
|
|
};
|
|
|
|
foreach (var pairRef in pairs)
|
|
{
|
|
_pairService.FindByIdAsync(pairRef.PairId, Arg.Any<CancellationToken>())
|
|
.Returns(CreateSecurityPair(pairRef));
|
|
}
|
|
|
|
var request = CreateValidationRequest(pairs);
|
|
|
|
// Act
|
|
var result = await _sut.RunAsync(request);
|
|
|
|
// Assert
|
|
result.MarkdownReport.Should().Contain("libfirst");
|
|
result.MarkdownReport.Should().Contain("libsecond");
|
|
result.MarkdownReport.Should().Contain("CVE-2024-1234");
|
|
result.MarkdownReport.Should().Contain("CVE-2024-5678");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Timeout and Cancellation Tests
|
|
|
|
[Fact]
|
|
public async Task RunAsync_Cancellation_ReturnsCancelledOrFailedResult()
|
|
{
|
|
// Arrange
|
|
var pairRef = CreatePairReference("pair-001", "CVE-2024-1234", "libexample");
|
|
var startedSemaphore = new SemaphoreSlim(0);
|
|
|
|
// Make FindByIdAsync slow to allow cancellation
|
|
_pairService.FindByIdAsync(pairRef.PairId, Arg.Any<CancellationToken>())
|
|
.Returns(async callInfo =>
|
|
{
|
|
startedSemaphore.Release();
|
|
await Task.Delay(5000, callInfo.Arg<CancellationToken>());
|
|
return CreateSecurityPair(pairRef);
|
|
});
|
|
|
|
var request = CreateValidationRequest([pairRef]);
|
|
using var cts = new CancellationTokenSource();
|
|
|
|
// Act
|
|
var runTask = _sut.RunAsync(request, cts.Token);
|
|
|
|
// Wait for the operation to actually start
|
|
await startedSemaphore.WaitAsync(TimeSpan.FromSeconds(5));
|
|
await cts.CancelAsync();
|
|
|
|
var result = await runTask;
|
|
|
|
// Assert - may complete as cancelled or failed depending on timing
|
|
result.Status.State.Should().BeOneOf(
|
|
ValidationState.Cancelled,
|
|
ValidationState.Failed,
|
|
ValidationState.Completed); // May complete if cancellation is too slow
|
|
|
|
// If completed, verify it handled the early return gracefully
|
|
result.Should().NotBeNull();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Configuration Tests
|
|
|
|
[Fact]
|
|
public async Task RunAsync_RespectsMaxParallelism()
|
|
{
|
|
// Arrange
|
|
var pairs = Enumerable.Range(1, 10)
|
|
.Select(i => CreatePairReference($"pair-{i:D3}", $"CVE-2024-{i:D4}", $"lib{i}"))
|
|
.ToImmutableArray();
|
|
|
|
var concurrentCalls = 0;
|
|
var maxConcurrentCalls = 0;
|
|
var lockObj = new object();
|
|
|
|
foreach (var pairRef in pairs)
|
|
{
|
|
_pairService.FindByIdAsync(pairRef.PairId, Arg.Any<CancellationToken>())
|
|
.Returns(async _ =>
|
|
{
|
|
lock (lockObj)
|
|
{
|
|
concurrentCalls++;
|
|
maxConcurrentCalls = Math.Max(maxConcurrentCalls, concurrentCalls);
|
|
}
|
|
await Task.Delay(50);
|
|
lock (lockObj)
|
|
{
|
|
concurrentCalls--;
|
|
}
|
|
return CreateSecurityPair(pairRef);
|
|
});
|
|
}
|
|
|
|
var request = new ValidationRunRequest
|
|
{
|
|
Pairs = pairs,
|
|
Matcher = CreateMatcherConfig(),
|
|
Metrics = CreateMetricsConfig(),
|
|
MaxParallelism = 2
|
|
};
|
|
|
|
// Act
|
|
await _sut.RunAsync(request);
|
|
|
|
// Assert - max parallelism should not exceed configured value
|
|
maxConcurrentCalls.Should().BeLessThanOrEqualTo(2);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Helper Methods
|
|
|
|
private static ValidationRunRequest CreateValidationRequest(
|
|
IEnumerable<SecurityPairReference> pairs)
|
|
{
|
|
return new ValidationRunRequest
|
|
{
|
|
Pairs = [.. pairs],
|
|
Matcher = CreateMatcherConfig(),
|
|
Metrics = CreateMetricsConfig()
|
|
};
|
|
}
|
|
|
|
private static MatcherConfiguration CreateMatcherConfig()
|
|
{
|
|
return new MatcherConfiguration
|
|
{
|
|
Algorithm = MatchingAlgorithm.Ensemble,
|
|
MinimumSimilarity = 0.85,
|
|
UseNameMatching = true,
|
|
UseStructuralMatching = true,
|
|
UseSemanticMatching = true
|
|
};
|
|
}
|
|
|
|
private static MetricsConfiguration CreateMetricsConfig()
|
|
{
|
|
return new MetricsConfiguration
|
|
{
|
|
ComputeMatchRate = true,
|
|
ComputeFalseNegativeRate = true,
|
|
VerifySbomStability = true,
|
|
SbomStabilityRuns = 3,
|
|
GenerateMismatchBuckets = true
|
|
};
|
|
}
|
|
|
|
private static SecurityPairReference CreatePairReference(
|
|
string pairId,
|
|
string cveId,
|
|
string packageName)
|
|
{
|
|
return new SecurityPairReference
|
|
{
|
|
PairId = pairId,
|
|
CveId = cveId,
|
|
PackageName = packageName,
|
|
VulnerableVersion = "1.0.0",
|
|
PatchedVersion = "1.0.1"
|
|
};
|
|
}
|
|
|
|
private static SecurityPair CreateSecurityPair(
|
|
SecurityPairReference pairRef,
|
|
int changedFunctionCount = 1)
|
|
{
|
|
var changedFunctions = Enumerable.Range(1, changedFunctionCount)
|
|
.Select(i => new ChangedFunction(
|
|
$"vuln_function_{i}",
|
|
VulnerableSize: 100 + i * 10,
|
|
PatchedSize: 120 + i * 10,
|
|
SizeDelta: 20,
|
|
ChangeType.Modified,
|
|
"Security fix"))
|
|
.ToImmutableArray();
|
|
|
|
return new SecurityPair
|
|
{
|
|
PairId = pairRef.PairId,
|
|
CveId = pairRef.CveId,
|
|
PackageName = pairRef.PackageName,
|
|
VulnerableVersion = pairRef.VulnerableVersion,
|
|
PatchedVersion = pairRef.PatchedVersion,
|
|
Distro = "debian",
|
|
VulnerableObservationId = $"obs-vuln-{pairRef.PairId}",
|
|
VulnerableDebugId = $"dbg-vuln-{pairRef.PairId}",
|
|
PatchedObservationId = $"obs-patch-{pairRef.PairId}",
|
|
PatchedDebugId = $"dbg-patch-{pairRef.PairId}",
|
|
AffectedFunctions = [new AffectedFunction(
|
|
"vulnerable_func",
|
|
VulnerableAddress: 0x1000,
|
|
PatchedAddress: 0x1000,
|
|
AffectedFunctionType.Vulnerable,
|
|
"Main vulnerability")],
|
|
ChangedFunctions = changedFunctions,
|
|
CreatedAt = DateTimeOffset.UtcNow
|
|
};
|
|
}
|
|
|
|
#endregion
|
|
}
|