sprints work.
This commit is contained in:
master
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.BinaryIndex.GroundTruth.Abstractions\StellaOps.BinaryIndex.GroundTruth.Abstractions.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,426 @@
|
||||
using System.Collections.Immutable;
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Abstractions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for AOC (Aggregation-Only Contract) write guard invariants.
|
||||
/// </summary>
|
||||
public class SymbolObservationWriteGuardTests
|
||||
{
|
||||
private readonly SymbolObservationWriteGuard _guard = new();
|
||||
|
||||
#region ValidateWrite Tests
|
||||
|
||||
[Fact]
|
||||
public void ValidateWrite_NewObservation_ReturnsProceed()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation();
|
||||
|
||||
// Act
|
||||
var result = _guard.ValidateWrite(observation, existingContentHash: null);
|
||||
|
||||
// Assert
|
||||
result.Should().Be(WriteDisposition.Proceed);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateWrite_IdenticalContentHash_ReturnsSkipIdentical()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation();
|
||||
var existingHash = observation.ContentHash;
|
||||
|
||||
// Act
|
||||
var result = _guard.ValidateWrite(observation, existingHash);
|
||||
|
||||
// Assert
|
||||
result.Should().Be(WriteDisposition.SkipIdentical);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateWrite_DifferentContentHash_ReturnsRejectMutation()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation();
|
||||
var existingHash = "sha256:differenthash";
|
||||
|
||||
// Act
|
||||
var result = _guard.ValidateWrite(observation, existingHash);
|
||||
|
||||
// Assert
|
||||
result.Should().Be(WriteDisposition.RejectMutation);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateWrite_CaseInsensitiveHashComparison_ReturnsSkipIdentical()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation();
|
||||
var existingHash = observation.ContentHash.ToUpperInvariant();
|
||||
|
||||
// Act
|
||||
var result = _guard.ValidateWrite(observation, existingHash);
|
||||
|
||||
// Assert
|
||||
result.Should().Be(WriteDisposition.SkipIdentical);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region EnsureValid - Required Fields Tests
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_ValidObservation_DoesNotThrow()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation();
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().NotThrow();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingObservationId_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with { ObservationId = "" };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingRequiredField))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "observationId"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingSourceId_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with { SourceId = "" };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingRequiredField))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "sourceId"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingDebugId_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with { DebugId = "" };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingRequiredField))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "debugId"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingBinaryName_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with { BinaryName = "" };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingRequiredField))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "binaryName"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingArchitecture_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with { Architecture = "" };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingRequiredField))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "architecture"));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region EnsureValid - Provenance Tests (GTAOC_001)
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingProvenance_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with { Provenance = null! };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingProvenance));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingProvenanceSourceId_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with
|
||||
{
|
||||
Provenance = CreateValidProvenance() with { SourceId = "" }
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingProvenance))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "provenance.sourceId"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingProvenanceDocumentUri_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with
|
||||
{
|
||||
Provenance = CreateValidProvenance() with { DocumentUri = "" }
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingProvenance))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "provenance.documentUri"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MissingProvenanceDocumentHash_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with
|
||||
{
|
||||
Provenance = CreateValidProvenance() with { DocumentHash = "" }
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingProvenance))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "provenance.documentHash"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_DefaultProvenanceFetchedAt_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with
|
||||
{
|
||||
Provenance = CreateValidProvenance() with { FetchedAt = default }
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.MissingProvenance))
|
||||
.Where(ex => ex.Violations.Any(v => v.Path == "provenance.fetchedAt"));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region EnsureValid - Content Hash Tests (GTAOC_004)
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_InvalidContentHash_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with
|
||||
{
|
||||
ContentHash = "sha256:invalidhash"
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.InvalidContentHash));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeContentHash_DeterministicForSameInput()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation();
|
||||
|
||||
// Act
|
||||
var hash1 = SymbolObservationWriteGuard.ComputeContentHash(observation);
|
||||
var hash2 = SymbolObservationWriteGuard.ComputeContentHash(observation);
|
||||
|
||||
// Assert
|
||||
hash1.Should().Be(hash2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeContentHash_DifferentForDifferentInput()
|
||||
{
|
||||
// Arrange
|
||||
var observation1 = CreateValidObservation();
|
||||
var observation2 = CreateValidObservation() with { DebugId = "different-debug-id" };
|
||||
|
||||
// Act
|
||||
var hash1 = SymbolObservationWriteGuard.ComputeContentHash(observation1);
|
||||
var hash2 = SymbolObservationWriteGuard.ComputeContentHash(observation2);
|
||||
|
||||
// Assert
|
||||
hash1.Should().NotBe(hash2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ComputeContentHash_StartsWithSha256Prefix()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation();
|
||||
|
||||
// Act
|
||||
var hash = SymbolObservationWriteGuard.ComputeContentHash(observation);
|
||||
|
||||
// Assert
|
||||
hash.Should().StartWith("sha256:");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region EnsureValid - Supersession Chain Tests (GTAOC_006)
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_SupersedesItself_ThrowsWithCorrectCode()
|
||||
{
|
||||
// Arrange
|
||||
var observationId = "groundtruth:test-source:build123:1";
|
||||
var observation = CreateValidObservation() with
|
||||
{
|
||||
ObservationId = observationId,
|
||||
SupersedesId = observationId
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Any(v => v.Code == AocViolationCodes.InvalidSupersession));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_ValidSupersession_DoesNotThrow()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with
|
||||
{
|
||||
ObservationId = "groundtruth:test-source:build123:2",
|
||||
SupersedesId = "groundtruth:test-source:build123:1"
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().NotThrow();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_NullSupersedes_DoesNotThrow()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with { SupersedesId = null };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().NotThrow();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Multiple Violations Tests
|
||||
|
||||
[Fact]
|
||||
public void EnsureValid_MultipleViolations_ReportsAll()
|
||||
{
|
||||
// Arrange
|
||||
var observation = CreateValidObservation() with
|
||||
{
|
||||
ObservationId = "",
|
||||
SourceId = "",
|
||||
DebugId = ""
|
||||
};
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _guard.EnsureValid(observation);
|
||||
act.Should().Throw<GroundTruthAocGuardException>()
|
||||
.Where(ex => ex.Violations.Count >= 3);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region AocViolation Record Tests
|
||||
|
||||
[Fact]
|
||||
public void AocViolation_RecordEquality()
|
||||
{
|
||||
// Arrange
|
||||
var v1 = new AocViolation(AocViolationCodes.MissingProvenance, "test", "path", AocViolationSeverity.Error);
|
||||
var v2 = new AocViolation(AocViolationCodes.MissingProvenance, "test", "path", AocViolationSeverity.Error);
|
||||
var v3 = new AocViolation(AocViolationCodes.MissingRequiredField, "test", "path", AocViolationSeverity.Error);
|
||||
|
||||
// Assert
|
||||
v1.Should().Be(v2);
|
||||
v1.Should().NotBe(v3);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static SymbolObservation CreateValidObservation()
|
||||
{
|
||||
var provenance = CreateValidProvenance();
|
||||
var symbols = ImmutableArray.Create(new ObservedSymbol
|
||||
{
|
||||
Name = "main",
|
||||
Address = 0x1000,
|
||||
Size = 100,
|
||||
Type = SymbolType.Function,
|
||||
Binding = SymbolBinding.Global
|
||||
});
|
||||
|
||||
var baseObservation = new SymbolObservation
|
||||
{
|
||||
ObservationId = "groundtruth:test-source:abcd1234:1",
|
||||
SourceId = "test-source",
|
||||
DebugId = "abcd1234",
|
||||
BinaryName = "test.so",
|
||||
Architecture = "x86_64",
|
||||
Symbols = symbols,
|
||||
SymbolCount = 1,
|
||||
Provenance = provenance,
|
||||
ContentHash = "", // Will be computed
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
// Compute and set the correct content hash
|
||||
var hash = SymbolObservationWriteGuard.ComputeContentHash(baseObservation);
|
||||
return baseObservation with { ContentHash = hash };
|
||||
}
|
||||
|
||||
private static ObservationProvenance CreateValidProvenance()
|
||||
{
|
||||
return new ObservationProvenance
|
||||
{
|
||||
SourceId = "test-source",
|
||||
DocumentUri = "https://example.com/test.elf",
|
||||
FetchedAt = DateTimeOffset.UtcNow.AddMinutes(-5),
|
||||
RecordedAt = DateTimeOffset.UtcNow,
|
||||
DocumentHash = "sha256:abc123",
|
||||
SignatureState = SignatureState.None
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Buildinfo.Configuration;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Buildinfo.Tests.Fixtures;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Buildinfo.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for Buildinfo connector.
|
||||
/// These tests require network access to buildinfos.debian.net.
|
||||
/// Skip in CI by setting SKIP_INTEGRATION_TESTS=true.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public class BuildinfoConnectorIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private ServiceProvider? _services;
|
||||
private readonly bool _skipTests;
|
||||
|
||||
public BuildinfoConnectorIntegrationTests()
|
||||
{
|
||||
_skipTests = Environment.GetEnvironmentVariable("SKIP_INTEGRATION_TESTS")?.ToLowerInvariant() == "true"
|
||||
|| Environment.GetEnvironmentVariable("CI")?.ToLowerInvariant() == "true";
|
||||
}
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
if (_skipTests)
|
||||
return Task.CompletedTask;
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug));
|
||||
services.AddBuildinfoConnector(opts =>
|
||||
{
|
||||
opts.Distributions = ["bookworm"];
|
||||
opts.Architectures = ["amd64"];
|
||||
opts.TimeoutSeconds = 60;
|
||||
opts.VerifySignatures = false; // Don't verify for integration tests
|
||||
});
|
||||
|
||||
_services = services.BuildServiceProvider();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_services?.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BuildinfoConnector_CanTestConnectivity()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<BuildinfoConnector>();
|
||||
|
||||
// Act
|
||||
var result = await connector.TestConnectivityAsync();
|
||||
|
||||
// Assert
|
||||
result.IsConnected.Should().BeTrue("Should be able to connect to buildinfos.debian.net");
|
||||
result.Latency.Should().BeLessThan(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BuildinfoConnector_CanGetMetadata()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<BuildinfoConnector>();
|
||||
|
||||
// Act
|
||||
var metadata = await connector.GetMetadataAsync();
|
||||
|
||||
// Assert
|
||||
metadata.SourceId.Should().Be("buildinfo-debian");
|
||||
metadata.DisplayName.Should().Contain("Debian");
|
||||
metadata.BaseUrl.Should().Contain("buildinfos.debian.net");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildinfoConnector_HasCorrectProperties()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<BuildinfoConnector>();
|
||||
|
||||
// Assert
|
||||
connector.SourceId.Should().Be("buildinfo-debian");
|
||||
connector.DisplayName.Should().Contain("Reproducible");
|
||||
connector.SupportedDistros.Should().Contain("debian");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BuildinfoConnector_FetchBuildinfo_ReturnsDataForKnownPackage()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<BuildinfoConnector>();
|
||||
|
||||
// Act - try to fetch a well-known package buildinfo
|
||||
// Note: This may fail if the exact version doesn't exist
|
||||
var result = await connector.FetchBuildinfoAsync(
|
||||
"coreutils",
|
||||
"9.1-1",
|
||||
"amd64");
|
||||
|
||||
// Assert - if found, verify structure
|
||||
if (result is not null)
|
||||
{
|
||||
result.Source.Should().Be("coreutils");
|
||||
result.Checksums.Should().NotBeEmpty();
|
||||
result.InstalledBuildDepends.Should().NotBeEmpty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides Skip functionality for xUnit when condition is true.
|
||||
/// </summary>
|
||||
public static class Skip
|
||||
{
|
||||
public static void If(bool condition, string reason)
|
||||
{
|
||||
if (condition)
|
||||
{
|
||||
throw new SkipException(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception to skip a test.
|
||||
/// </summary>
|
||||
public class SkipException : Exception
|
||||
{
|
||||
public SkipException(string reason) : base(reason) { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test meter factory for diagnostics.
|
||||
/// </summary>
|
||||
internal sealed class TestMeterFactory : System.Diagnostics.Metrics.IMeterFactory
|
||||
{
|
||||
public System.Diagnostics.Metrics.Meter Create(System.Diagnostics.Metrics.MeterOptions options)
|
||||
=> new(options.Name, options.Version);
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
using FluentAssertions;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Buildinfo.Internal;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Buildinfo.Tests.Fixtures;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Buildinfo.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for BuildinfoParser using deterministic fixtures.
|
||||
/// </summary>
|
||||
public class BuildinfoParserTests
|
||||
{
|
||||
private readonly BuildinfoParser _parser = new();
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleCurlBuildinfo_ParsesSourceAndVersion()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleBuildinfoCurl();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.Source.Should().Be(FixtureConstants.SampleSourcePackageCurl);
|
||||
result.Version.Should().Be(FixtureConstants.SampleVersionCurl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleCurlBuildinfo_ExtractsBinaries()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleBuildinfoCurl();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.Binaries.Should().Contain(FixtureConstants.ExpectedBinaryCurl);
|
||||
result.Binaries.Should().Contain(FixtureConstants.ExpectedBinaryLibcurl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleCurlBuildinfo_ExtractsChecksums()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleBuildinfoCurl();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.Checksums.Should().HaveCountGreaterThanOrEqualTo(2);
|
||||
result.Checksums.Should().Contain(c => c.Algorithm == "sha256");
|
||||
result.Checksums.Should().Contain(c => c.Filename.Contains("curl"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleCurlBuildinfo_ExtractsBuildMetadata()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleBuildinfoCurl();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.BuildOrigin.Should().Be("debian");
|
||||
result.BuildArchitecture.Should().Be(FixtureConstants.SampleArchitectureAmd64);
|
||||
result.BuildDate.Should().NotBeNull();
|
||||
result.BuildPath.Should().StartWith("/build/");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleCurlBuildinfo_ExtractsBuildDependencies()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleBuildinfoCurl();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.InstalledBuildDepends.Should().HaveCountGreaterThanOrEqualTo(2);
|
||||
result.InstalledBuildDepends.Should().Contain(d => d.Package == "gcc");
|
||||
result.InstalledBuildDepends.Should().Contain(d => d.Package == "libc6");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleCurlBuildinfo_ExtractsEnvironment()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleBuildinfoCurl();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.Environment.Should().ContainKey("DEB_BUILD_OPTIONS");
|
||||
result.Environment.Should().ContainKey("LANG");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleSignedBuildinfo_DetectsSignature()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSignedBuildinfo();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.IsSigned.Should().BeTrue();
|
||||
result.Source.Should().Be(FixtureConstants.SampleSourcePackageOpenssl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleSignedBuildinfo_StripsSignatureAndParses()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSignedBuildinfo();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.Version.Should().Be(FixtureConstants.SampleVersionOpenssl);
|
||||
result.Binaries.Should().Contain(FixtureConstants.ExpectedBinaryOpenssl);
|
||||
result.Binaries.Should().Contain(FixtureConstants.ExpectedBinaryLibssl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_UnsignedBuildinfo_ReportsNotSigned()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleBuildinfoCurl();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.IsSigned.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_MissingRequiredSource_Throws()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Format: 1.0
|
||||
Version: 1.0
|
||||
Binary: test
|
||||
""";
|
||||
|
||||
// Act
|
||||
var act = () => _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
act.Should().Throw<FormatException>()
|
||||
.WithMessage("*Source*");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_MissingRequiredVersion_Throws()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Format: 1.0
|
||||
Source: test
|
||||
Binary: test
|
||||
""";
|
||||
|
||||
// Act
|
||||
var act = () => _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
act.Should().Throw<FormatException>()
|
||||
.WithMessage("*Version*");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_DependencyWithVersionConstraint_ParsesCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Format: 1.0
|
||||
Source: test
|
||||
Version: 1.0
|
||||
Installed-Build-Depends:
|
||||
gcc (= 12.2.0-14),
|
||||
libc6 (>= 2.36)
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.InstalledBuildDepends.Should().HaveCount(2);
|
||||
result.InstalledBuildDepends[0].Package.Should().Be("gcc");
|
||||
result.InstalledBuildDepends[0].Version.Should().Be("12.2.0-14");
|
||||
result.InstalledBuildDepends[1].Package.Should().Be("libc6");
|
||||
result.InstalledBuildDepends[1].Version.Should().Be("2.36");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_DependencyWithArchitecture_ParsesCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Format: 1.0
|
||||
Source: test
|
||||
Version: 1.0
|
||||
Installed-Build-Depends:
|
||||
libc6:amd64 (= 2.36-9)
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.InstalledBuildDepends.Should().HaveCount(1);
|
||||
result.InstalledBuildDepends[0].Package.Should().Be("libc6");
|
||||
result.InstalledBuildDepends[0].Architecture.Should().Be("amd64");
|
||||
result.InstalledBuildDepends[0].Version.Should().Be("2.36-9");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_ChecksumLines_ParsesSizeCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Format: 1.0
|
||||
Source: test
|
||||
Version: 1.0
|
||||
Checksums-Sha256:
|
||||
abc123 12345678 test_1.0_amd64.deb
|
||||
def456 98765432 test-dev_1.0_amd64.deb
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.Checksums.Should().HaveCount(2);
|
||||
result.Checksums[0].Size.Should().Be(12345678);
|
||||
result.Checksums[1].Size.Should().Be(98765432);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_ContinuationLines_HandlesCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Format: 1.0
|
||||
Source: test
|
||||
Version: 1.0
|
||||
Binary: pkg1
|
||||
pkg2
|
||||
pkg3
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.Binaries.Should().Contain("pkg1");
|
||||
result.Binaries.Should().Contain("pkg2");
|
||||
result.Binaries.Should().Contain("pkg3");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_Rfc2822Date_ParsesCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Format: 1.0
|
||||
Source: test
|
||||
Version: 1.0
|
||||
Build-Date: Mon, 15 Jan 2024 10:30:00 +0000
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.BuildDate.Should().NotBeNull();
|
||||
result.BuildDate!.Value.Year.Should().Be(2024);
|
||||
result.BuildDate!.Value.Month.Should().Be(1);
|
||||
result.BuildDate!.Value.Day.Should().Be(15);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_EmptyContent_ThrowsArgumentNullException()
|
||||
{
|
||||
// Act
|
||||
var act = () => _parser.Parse(null!);
|
||||
|
||||
// Assert
|
||||
act.Should().Throw<ArgumentNullException>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_DashEscapedContent_UnescapesCorrectly()
|
||||
{
|
||||
// PGP clearsign escapes lines starting with - as "- -"
|
||||
// Arrange
|
||||
var content = """
|
||||
-----BEGIN PGP SIGNED MESSAGE-----
|
||||
Hash: SHA512
|
||||
|
||||
Format: 1.0
|
||||
Source: test
|
||||
Version: 1.0-rc1
|
||||
-----BEGIN PGP SIGNATURE-----
|
||||
abc123
|
||||
-----END PGP SIGNATURE-----
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content);
|
||||
|
||||
// Assert
|
||||
result.IsSigned.Should().BeTrue();
|
||||
result.Source.Should().Be("test");
|
||||
result.Version.Should().Be("1.0-rc1");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
using System.Reflection;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Buildinfo.Tests.Fixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Provides access to deterministic test fixtures for offline testing.
|
||||
/// </summary>
|
||||
public static class FixtureProvider
|
||||
{
|
||||
private static readonly string FixturesPath;
|
||||
|
||||
static FixtureProvider()
|
||||
{
|
||||
var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
|
||||
FixturesPath = Path.Combine(assemblyDir, "Fixtures");
|
||||
|
||||
// Also try the source directory for development
|
||||
if (!Directory.Exists(FixturesPath))
|
||||
{
|
||||
var sourceDir = FindSourceFixturesDirectory();
|
||||
if (sourceDir is not null)
|
||||
{
|
||||
FixturesPath = sourceDir;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get sample .buildinfo file content for curl package.
|
||||
/// </summary>
|
||||
public static string GetSampleBuildinfoCurl()
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, "curl_7.88.1-10_amd64.buildinfo");
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
// Return inline fixture if file doesn't exist
|
||||
return SampleBuildinfoContent;
|
||||
}
|
||||
return File.ReadAllText(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get sample signed .buildinfo file content.
|
||||
/// </summary>
|
||||
public static string GetSampleSignedBuildinfo()
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, "openssl_3.0.11-1_amd64.buildinfo.asc");
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return SampleSignedBuildinfoContent;
|
||||
}
|
||||
return File.ReadAllText(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get a fixture file as a stream.
|
||||
/// </summary>
|
||||
public static Stream GetFixtureStream(string name)
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, name);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
throw new FileNotFoundException($"Fixture not found: {path}");
|
||||
}
|
||||
return File.OpenRead(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if a fixture exists.
|
||||
/// </summary>
|
||||
public static bool FixtureExists(string name)
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, name);
|
||||
return File.Exists(path);
|
||||
}
|
||||
|
||||
private static string? FindSourceFixturesDirectory()
|
||||
{
|
||||
var dir = Directory.GetCurrentDirectory();
|
||||
while (dir is not null)
|
||||
{
|
||||
var candidate = Path.Combine(dir, "src", "BinaryIndex", "__Tests",
|
||||
"StellaOps.BinaryIndex.GroundTruth.Buildinfo.Tests", "Fixtures");
|
||||
if (Directory.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
dir = Directory.GetParent(dir)?.FullName;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inline sample buildinfo content for deterministic testing.
|
||||
/// </summary>
|
||||
private const string SampleBuildinfoContent = """
|
||||
Format: 1.0
|
||||
Source: curl
|
||||
Binary: curl libcurl4 libcurl4-openssl-dev
|
||||
Architecture: amd64 source
|
||||
Version: 7.88.1-10
|
||||
Checksums-Sha256:
|
||||
abc123def456789012345678901234567890123456789012345678901234 12345 curl_7.88.1-10_amd64.deb
|
||||
def456abc789012345678901234567890123456789012345678901234567 23456 libcurl4_7.88.1-10_amd64.deb
|
||||
Build-Origin: debian
|
||||
Build-Architecture: amd64
|
||||
Build-Date: Mon, 15 Jan 2024 10:30:00 +0000
|
||||
Build-Path: /build/curl-xyz123
|
||||
Installed-Build-Depends:
|
||||
gcc (= 12.2.0-14),
|
||||
libc6 (= 2.36-9),
|
||||
libssl-dev (= 3.0.11-1),
|
||||
zlib1g-dev (= 1:1.2.13.dfsg-1)
|
||||
Environment:
|
||||
DEB_BUILD_OPTIONS="parallel=8"
|
||||
LANG="C.UTF-8"
|
||||
SOURCE_DATE_EPOCH="1705315800"
|
||||
""";
|
||||
|
||||
/// <summary>
|
||||
/// Inline sample signed buildinfo content.
|
||||
/// </summary>
|
||||
private const string SampleSignedBuildinfoContent = """
|
||||
-----BEGIN PGP SIGNED MESSAGE-----
|
||||
Hash: SHA512
|
||||
|
||||
Format: 1.0
|
||||
Source: openssl
|
||||
Binary: openssl libssl3 libssl-dev
|
||||
Architecture: amd64 source
|
||||
Version: 3.0.11-1
|
||||
Checksums-Sha256:
|
||||
fedcba9876543210fedcba9876543210fedcba9876543210fedcba98765 45678 openssl_3.0.11-1_amd64.deb
|
||||
012345abcdef6789012345abcdef6789012345abcdef6789012345abcdef 56789 libssl3_3.0.11-1_amd64.deb
|
||||
Build-Origin: debian
|
||||
Build-Architecture: amd64
|
||||
Build-Date: Tue, 16 Jan 2024 14:00:00 +0000
|
||||
Build-Path: /build/openssl-abc456
|
||||
Installed-Build-Depends:
|
||||
gcc (= 12.2.0-14),
|
||||
libc6 (= 2.36-9),
|
||||
perl (= 5.36.0-7)
|
||||
Environment:
|
||||
DEB_BUILD_OPTIONS="nocheck"
|
||||
LANG="C.UTF-8"
|
||||
-----BEGIN PGP SIGNATURE-----
|
||||
|
||||
iQIzBAEBCgAdFiEE1234567890abcdef1234567890abcdef12345FiQI
|
||||
ZABC123/ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC1
|
||||
23ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC123ABC1
|
||||
=wxYz
|
||||
-----END PGP SIGNATURE-----
|
||||
""";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test fixture constants for buildinfo tests.
|
||||
/// </summary>
|
||||
public static class FixtureConstants
|
||||
{
|
||||
// Sample package info
|
||||
public const string SampleSourcePackageCurl = "curl";
|
||||
public const string SampleVersionCurl = "7.88.1-10";
|
||||
public const string SampleArchitectureAmd64 = "amd64";
|
||||
|
||||
public const string SampleSourcePackageOpenssl = "openssl";
|
||||
public const string SampleVersionOpenssl = "3.0.11-1";
|
||||
|
||||
// Expected binary names
|
||||
public const string ExpectedBinaryCurl = "curl";
|
||||
public const string ExpectedBinaryLibcurl = "libcurl4";
|
||||
public const string ExpectedBinaryOpenssl = "openssl";
|
||||
public const string ExpectedBinaryLibssl = "libssl3";
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.BinaryIndex.GroundTruth.Buildinfo\StellaOps.BinaryIndex.GroundTruth.Buildinfo.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="Fixtures\**\*">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,336 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Ddeb.Configuration;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Ddeb.Internal;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Ddeb.Tests.Fixtures;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Ddeb.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for Ddeb connector.
|
||||
/// These tests require network access to Ubuntu ddebs repository.
|
||||
/// Skip in CI by setting SKIP_INTEGRATION_TESTS=true.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public class DdebConnectorIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private ServiceProvider? _services;
|
||||
private readonly bool _skipTests;
|
||||
|
||||
public DdebConnectorIntegrationTests()
|
||||
{
|
||||
_skipTests = Environment.GetEnvironmentVariable("SKIP_INTEGRATION_TESTS")?.ToLowerInvariant() == "true"
|
||||
|| Environment.GetEnvironmentVariable("CI")?.ToLowerInvariant() == "true";
|
||||
}
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
if (_skipTests)
|
||||
return Task.CompletedTask;
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug));
|
||||
services.AddDdebConnector(opts =>
|
||||
{
|
||||
opts.Distributions = ["jammy"];
|
||||
opts.Components = ["main"];
|
||||
opts.Architectures = ["amd64"];
|
||||
opts.TimeoutSeconds = 60;
|
||||
});
|
||||
|
||||
_services = services.BuildServiceProvider();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_services?.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DdebConnector_CanFetchPackagesIndex()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var httpClientFactory = _services!.GetRequiredService<IHttpClientFactory>();
|
||||
var client = httpClientFactory.CreateClient(DdebOptions.HttpClientName);
|
||||
|
||||
// Act
|
||||
var response = await client.GetAsync("dists/jammy/main/debug/binary-amd64/Packages.gz");
|
||||
|
||||
// Assert
|
||||
response.IsSuccessStatusCode.Should().BeTrue("Should be able to fetch Packages.gz");
|
||||
response.Content.Headers.ContentLength.Should().BeGreaterThan(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DdebConnector_CanConnectToUbuntuDdebs()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<DdebConnector>();
|
||||
|
||||
// Act - just test that the connector can be instantiated and accessed
|
||||
connector.SourceId.Should().Be("ddeb-ubuntu");
|
||||
connector.DisplayName.Should().Contain("Ubuntu");
|
||||
connector.SupportedDistros.Should().Contain("ubuntu");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for Packages index parser using deterministic fixtures.
|
||||
/// </summary>
|
||||
public class PackagesIndexParserTests
|
||||
{
|
||||
private readonly PackagesIndexParser _parser = new();
|
||||
|
||||
[Fact]
|
||||
public void Parse_FixturePackagesIndex_ParsesAllPackages()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetPackagesIndexJammyMainAmd64();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content,
|
||||
FixtureConstants.SampleDistribution,
|
||||
FixtureConstants.SampleComponent,
|
||||
FixtureConstants.SampleArchitecture);
|
||||
|
||||
// Assert
|
||||
result.Should().HaveCountGreaterThan(0, "Fixture should contain packages");
|
||||
result.Should().Contain(p => p.PackageName == FixtureConstants.SamplePackageNameLibc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_FixtureLibcPackage_HasCorrectFields()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetPackagesIndexJammyMainAmd64();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content,
|
||||
FixtureConstants.SampleDistribution,
|
||||
FixtureConstants.SampleComponent,
|
||||
FixtureConstants.SampleArchitecture);
|
||||
|
||||
// Assert
|
||||
var libc = result.FirstOrDefault(p => p.PackageName == FixtureConstants.SamplePackageNameLibc);
|
||||
libc.Should().NotBeNull();
|
||||
libc!.Version.Should().Be(FixtureConstants.SamplePackageVersionLibc);
|
||||
libc.Distribution.Should().Be(FixtureConstants.SampleDistribution);
|
||||
libc.Component.Should().Be(FixtureConstants.SampleComponent);
|
||||
libc.Architecture.Should().Be(FixtureConstants.SampleArchitecture);
|
||||
libc.PoolUrl.Should().StartWith("/pool/main/g/glibc/");
|
||||
libc.Size.Should().BeGreaterThan(0);
|
||||
libc.Sha256.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_FixtureLinuxKernel_HasLargeSize()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetPackagesIndexJammyMainAmd64();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content,
|
||||
FixtureConstants.SampleDistribution,
|
||||
FixtureConstants.SampleComponent,
|
||||
FixtureConstants.SampleArchitecture);
|
||||
|
||||
// Assert
|
||||
var kernel = result.FirstOrDefault(p => p.PackageName.Contains("linux-image"));
|
||||
kernel.Should().NotBeNull("Fixture should contain Linux kernel package");
|
||||
kernel!.Size.Should().BeGreaterThan(100_000_000, "Kernel debug symbols are large");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_ValidPackageStanza_ExtractsFields()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Package: libc6-dbgsym
|
||||
Source: glibc
|
||||
Version: 2.35-0ubuntu3.1
|
||||
Architecture: amd64
|
||||
Filename: pool/main/g/glibc/libc6-dbgsym_2.35-0ubuntu3.1_amd64.ddeb
|
||||
Size: 10485760
|
||||
SHA256: abc123def456
|
||||
Description: debug symbols for libc6
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, "jammy", "main", "amd64");
|
||||
|
||||
// Assert
|
||||
result.Should().HaveCount(1);
|
||||
var pkg = result[0];
|
||||
pkg.PackageName.Should().Be("libc6-dbgsym");
|
||||
pkg.Version.Should().Be("2.35-0ubuntu3.1");
|
||||
pkg.Architecture.Should().Be("amd64");
|
||||
pkg.PoolUrl.Should().Be("/pool/main/g/glibc/libc6-dbgsym_2.35-0ubuntu3.1_amd64.ddeb");
|
||||
pkg.Size.Should().Be(10485760);
|
||||
pkg.Sha256.Should().Be("abc123def456");
|
||||
pkg.Distribution.Should().Be("jammy");
|
||||
pkg.Component.Should().Be("main");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_MultiplePackages_ParsesAll()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Package: pkg1-dbgsym
|
||||
Version: 1.0
|
||||
Filename: pool/main/p/pkg1/pkg1-dbgsym_1.0_amd64.ddeb
|
||||
|
||||
Package: pkg2-dbgsym
|
||||
Version: 2.0
|
||||
Filename: pool/main/p/pkg2/pkg2-dbgsym_2.0_amd64.ddeb
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, "jammy", "main", "amd64");
|
||||
|
||||
// Assert
|
||||
result.Should().HaveCount(2);
|
||||
result[0].PackageName.Should().Be("pkg1-dbgsym");
|
||||
result[1].PackageName.Should().Be("pkg2-dbgsym");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_MissingRequiredFields_SkipsPackage()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Package: incomplete-pkg
|
||||
Version: 1.0
|
||||
"""; // Missing Filename
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, "jammy", "main", "amd64");
|
||||
|
||||
// Assert
|
||||
result.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_ContinuationLines_HandlesCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Package: test-dbgsym
|
||||
Version: 1.0
|
||||
Filename: pool/main/t/test/test-dbgsym_1.0_amd64.ddeb
|
||||
Description: This is a long
|
||||
description that spans
|
||||
multiple lines
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, "jammy", "main", "amd64");
|
||||
|
||||
// Assert
|
||||
result.Should().HaveCount(1);
|
||||
result[0].Description.Should().Contain("multiple lines");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_EmptyContent_ReturnsEmptyList()
|
||||
{
|
||||
// Act
|
||||
var result = _parser.Parse("", "jammy", "main", "amd64");
|
||||
|
||||
// Assert
|
||||
result.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_InvalidSize_DefaultsToZero()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
Package: test-dbgsym
|
||||
Version: 1.0
|
||||
Filename: pool/main/t/test/test-dbgsym_1.0_amd64.ddeb
|
||||
Size: not-a-number
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, "jammy", "main", "amd64");
|
||||
|
||||
// Assert
|
||||
result.Should().HaveCount(1);
|
||||
result[0].Size.Should().Be(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for deb package extractor.
|
||||
/// </summary>
|
||||
public class DebPackageExtractorTests
|
||||
{
|
||||
[Fact]
|
||||
public void Extractor_PayloadIdOverload_ThrowsNotImplemented()
|
||||
{
|
||||
// Arrange
|
||||
var logger = new LoggerFactory().CreateLogger<DebPackageExtractor>();
|
||||
var diagnostics = new DdebDiagnostics(new TestMeterFactory());
|
||||
var extractor = new DebPackageExtractor(logger, diagnostics);
|
||||
|
||||
// Act & Assert
|
||||
var act = async () => await extractor.ExtractAsync(Guid.NewGuid());
|
||||
act.Should().ThrowAsync<NotImplementedException>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Extractor_InvalidArArchive_Throws()
|
||||
{
|
||||
// Arrange
|
||||
var logger = new LoggerFactory().CreateLogger<DebPackageExtractor>();
|
||||
var diagnostics = new DdebDiagnostics(new TestMeterFactory());
|
||||
var extractor = new DebPackageExtractor(logger, diagnostics);
|
||||
using var stream = new MemoryStream("not an ar archive"u8.ToArray());
|
||||
|
||||
// Act & Assert
|
||||
var act = async () => await extractor.ExtractAsync(stream);
|
||||
await act.Should().ThrowAsync<InvalidDataException>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test meter factory for diagnostics.
|
||||
/// </summary>
|
||||
internal sealed class TestMeterFactory : System.Diagnostics.Metrics.IMeterFactory
|
||||
{
|
||||
public System.Diagnostics.Metrics.Meter Create(System.Diagnostics.Metrics.MeterOptions options)
|
||||
=> new(options.Name, options.Version);
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides Skip functionality for xUnit when condition is true.
|
||||
/// </summary>
|
||||
public static class Skip
|
||||
{
|
||||
public static void If(bool condition, string reason)
|
||||
{
|
||||
if (condition)
|
||||
{
|
||||
throw new SkipException(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception to skip a test.
|
||||
/// </summary>
|
||||
public class SkipException : Exception
|
||||
{
|
||||
public SkipException(string reason) : base(reason) { }
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using System.Reflection;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Ddeb.Tests.Fixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Provides access to deterministic test fixtures for offline testing.
|
||||
/// </summary>
|
||||
public static class FixtureProvider
|
||||
{
|
||||
private static readonly string FixturesPath;
|
||||
|
||||
static FixtureProvider()
|
||||
{
|
||||
var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
|
||||
FixturesPath = Path.Combine(assemblyDir, "Fixtures");
|
||||
|
||||
// Also try the source directory for development
|
||||
if (!Directory.Exists(FixturesPath))
|
||||
{
|
||||
var sourceDir = FindSourceFixturesDirectory();
|
||||
if (sourceDir is not null)
|
||||
{
|
||||
FixturesPath = sourceDir;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get the sample Packages index content for Jammy main amd64.
|
||||
/// </summary>
|
||||
public static string GetPackagesIndexJammyMainAmd64()
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, "packages_index_jammy_main_amd64.txt");
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
throw new FileNotFoundException($"Fixture not found: {path}. Run tests from the project directory or ensure fixtures are copied to output.");
|
||||
}
|
||||
return File.ReadAllText(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get a fixture file as a stream.
|
||||
/// </summary>
|
||||
public static Stream GetFixtureStream(string name)
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, name);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
throw new FileNotFoundException($"Fixture not found: {path}");
|
||||
}
|
||||
return File.OpenRead(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if a fixture exists.
|
||||
/// </summary>
|
||||
public static bool FixtureExists(string name)
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, name);
|
||||
return File.Exists(path);
|
||||
}
|
||||
|
||||
private static string? FindSourceFixturesDirectory()
|
||||
{
|
||||
var dir = Directory.GetCurrentDirectory();
|
||||
while (dir is not null)
|
||||
{
|
||||
var candidate = Path.Combine(dir, "src", "BinaryIndex", "__Tests",
|
||||
"StellaOps.BinaryIndex.GroundTruth.Ddeb.Tests", "Fixtures");
|
||||
if (Directory.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
dir = Directory.GetParent(dir)?.FullName;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test fixture constants.
|
||||
/// </summary>
|
||||
public static class FixtureConstants
|
||||
{
|
||||
// Sample build IDs (hex strings)
|
||||
public const string SampleBuildIdLibc = "a27f9be2a0dc0e9bd63eba6daf42be012bb1be99";
|
||||
public const string SampleBuildIdBash = "b38e0ca1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7";
|
||||
public const string SampleBuildIdSsl = "c49f1db2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8";
|
||||
|
||||
// Sample package info
|
||||
public const string SamplePackageNameLibc = "libc6-dbgsym";
|
||||
public const string SamplePackageVersionLibc = "2.35-0ubuntu3.1";
|
||||
public const string SampleDistribution = "jammy";
|
||||
public const string SampleComponent = "main";
|
||||
public const string SampleArchitecture = "amd64";
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
Package: adduser-dbgsym
|
||||
Source: adduser
|
||||
Version: 3.118ubuntu5
|
||||
Installed-Size: 12
|
||||
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
|
||||
Architecture: all
|
||||
Filename: pool/main/a/adduser/adduser-dbgsym_3.118ubuntu5_all.ddeb
|
||||
Size: 2624
|
||||
SHA256: 2c9b4f6d3e8a1b0c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c
|
||||
Description: debug symbols for adduser
|
||||
|
||||
Package: apt-dbgsym
|
||||
Source: apt
|
||||
Version: 2.4.9
|
||||
Installed-Size: 456
|
||||
Maintainer: APT Development Team <deity@lists.debian.org>
|
||||
Architecture: amd64
|
||||
Filename: pool/main/a/apt/apt-dbgsym_2.4.9_amd64.ddeb
|
||||
Size: 1048576
|
||||
SHA256: 3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e
|
||||
Description: debug symbols for apt
|
||||
|
||||
Package: bash-dbgsym
|
||||
Source: bash
|
||||
Version: 5.1-6ubuntu1
|
||||
Installed-Size: 1024
|
||||
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
|
||||
Architecture: amd64
|
||||
Filename: pool/main/b/bash/bash-dbgsym_5.1-6ubuntu1_amd64.ddeb
|
||||
Size: 2097152
|
||||
SHA256: 4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f
|
||||
Description: debug symbols for bash
|
||||
|
||||
Package: libc6-dbgsym
|
||||
Source: glibc
|
||||
Version: 2.35-0ubuntu3.1
|
||||
Installed-Size: 14336
|
||||
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
|
||||
Architecture: amd64
|
||||
Filename: pool/main/g/glibc/libc6-dbgsym_2.35-0ubuntu3.1_amd64.ddeb
|
||||
Size: 10485760
|
||||
SHA256: 5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a
|
||||
Description: debug symbols for GNU C Library: Shared libraries
|
||||
Contains debugging symbols for the GNU C Library packages.
|
||||
|
||||
Package: libssl3-dbgsym
|
||||
Source: openssl
|
||||
Version: 3.0.2-0ubuntu1.10
|
||||
Installed-Size: 4096
|
||||
Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
|
||||
Architecture: amd64
|
||||
Filename: pool/main/o/openssl/libssl3-dbgsym_3.0.2-0ubuntu1.10_amd64.ddeb
|
||||
Size: 4194304
|
||||
SHA256: 6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b
|
||||
Description: debug symbols for OpenSSL SSL/TLS library
|
||||
|
||||
Package: linux-image-5.15.0-91-generic-dbgsym
|
||||
Source: linux
|
||||
Version: 5.15.0-91.101
|
||||
Installed-Size: 819200
|
||||
Maintainer: Ubuntu Kernel Team <kernel-team@lists.ubuntu.com>
|
||||
Architecture: amd64
|
||||
Filename: pool/main/l/linux/linux-image-5.15.0-91-generic-dbgsym_5.15.0-91.101_amd64.ddeb
|
||||
Size: 943718400
|
||||
SHA256: 7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c
|
||||
Description: debug symbols for Linux kernel image
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.BinaryIndex.GroundTruth.Ddeb\StellaOps.BinaryIndex.GroundTruth.Ddeb.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="Fixtures\**\*">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,175 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Abstractions;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Debuginfod.Configuration;
|
||||
using StellaOps.BinaryIndex.GroundTruth.Debuginfod.Internal;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.Debuginfod.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for Debuginfod connector.
|
||||
/// These tests require network access to real debuginfod servers.
|
||||
/// Skip in CI by setting SKIP_INTEGRATION_TESTS=true.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public class DebuginfodConnectorIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private ServiceProvider? _services;
|
||||
private readonly bool _skipTests;
|
||||
|
||||
public DebuginfodConnectorIntegrationTests()
|
||||
{
|
||||
_skipTests = Environment.GetEnvironmentVariable("SKIP_INTEGRATION_TESTS")?.ToLowerInvariant() == "true"
|
||||
|| Environment.GetEnvironmentVariable("CI")?.ToLowerInvariant() == "true";
|
||||
}
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
if (_skipTests)
|
||||
return Task.CompletedTask;
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug));
|
||||
services.AddDebuginfodConnector(opts =>
|
||||
{
|
||||
opts.BaseUrl = new Uri("https://debuginfod.fedoraproject.org");
|
||||
opts.TimeoutSeconds = 30;
|
||||
});
|
||||
|
||||
_services = services.BuildServiceProvider();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_services?.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DebuginfodConnector_CanConnectToFedora()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<DebuginfodConnector>();
|
||||
|
||||
// Act
|
||||
var result = await ((ISymbolSourceCapability)connector).TestConnectivityAsync();
|
||||
|
||||
// Assert
|
||||
result.IsConnected.Should().BeTrue("Fedora debuginfod should be reachable");
|
||||
result.Latency.Should().BeLessThan(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DebuginfodConnector_CanFetchKnownBuildId()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<DebuginfodConnector>();
|
||||
|
||||
// Well-known build ID from glibc (update if needed)
|
||||
// This is a commonly available debug binary
|
||||
var knownBuildId = "a27f9be2a0dc0e9bd63eba6daf42be012bb1be99"; // glibc example
|
||||
|
||||
// Act
|
||||
var result = await ((ISymbolSourceCapability)connector).FetchByDebugIdAsync(knownBuildId);
|
||||
|
||||
// Assert - may be null if specific build ID not available
|
||||
// This test primarily validates the fetch mechanism works
|
||||
// In production, use a guaranteed-available build ID
|
||||
if (result is not null)
|
||||
{
|
||||
result.DebugId.Should().NotBeNullOrEmpty();
|
||||
result.BinaryName.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DebuginfodConnector_ReturnsNullForUnknownBuildId()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<DebuginfodConnector>();
|
||||
var unknownBuildId = "0000000000000000000000000000000000000000";
|
||||
|
||||
// Act
|
||||
var result = await ((ISymbolSourceCapability)connector).FetchByDebugIdAsync(unknownBuildId);
|
||||
|
||||
// Assert
|
||||
result.Should().BeNull("Unknown build ID should return null");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for ELF/DWARF parser using local fixtures.
|
||||
/// </summary>
|
||||
public class ElfDwarfParserTests
|
||||
{
|
||||
[Fact]
|
||||
public void Parser_ThrowsOnInvalidStream()
|
||||
{
|
||||
// Arrange
|
||||
var logger = new LoggerFactory().CreateLogger<ElfDwarfParser>();
|
||||
var parser = new ElfDwarfParser(logger);
|
||||
using var stream = new MemoryStream([1, 2, 3, 4]); // Invalid ELF
|
||||
|
||||
// Act & Assert
|
||||
var act = async () => await parser.ParseSymbolsAsync(stream);
|
||||
act.Should().ThrowAsync<Exception>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Parser_ExtractBuildId_ReturnsNullForNonElf()
|
||||
{
|
||||
// Arrange
|
||||
var logger = new LoggerFactory().CreateLogger<ElfDwarfParser>();
|
||||
var parser = new ElfDwarfParser(logger);
|
||||
using var stream = new MemoryStream("not an elf file"u8.ToArray());
|
||||
|
||||
// Act
|
||||
var result = await parser.ExtractBuildIdAsync(stream);
|
||||
|
||||
// Assert
|
||||
result.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parser_PayloadIdOverload_ThrowsNotImplemented()
|
||||
{
|
||||
// Arrange
|
||||
var logger = new LoggerFactory().CreateLogger<ElfDwarfParser>();
|
||||
var parser = new ElfDwarfParser(logger);
|
||||
|
||||
// Act & Assert
|
||||
var act = async () => await parser.ParseSymbolsAsync(Guid.NewGuid());
|
||||
act.Should().ThrowAsync<NotImplementedException>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides Skip functionality for xUnit when condition is true.
|
||||
/// </summary>
|
||||
public static class Skip
|
||||
{
|
||||
public static void If(bool condition, string reason)
|
||||
{
|
||||
if (condition)
|
||||
{
|
||||
throw new SkipException(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception to skip a test.
|
||||
/// </summary>
|
||||
public class SkipException : Exception
|
||||
{
|
||||
public SkipException(string reason) : base(reason) { }
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.BinaryIndex.GroundTruth.Debuginfod\StellaOps.BinaryIndex.GroundTruth.Debuginfod.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,189 @@
|
||||
using System.Reflection;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.SecDb.Tests.Fixtures;
|
||||
|
||||
/// <summary>
|
||||
/// Provides access to deterministic test fixtures for offline testing.
|
||||
/// </summary>
|
||||
public static class FixtureProvider
|
||||
{
|
||||
private static readonly string FixturesPath;
|
||||
|
||||
static FixtureProvider()
|
||||
{
|
||||
var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
|
||||
FixturesPath = Path.Combine(assemblyDir, "Fixtures");
|
||||
|
||||
// Also try the source directory for development
|
||||
if (!Directory.Exists(FixturesPath))
|
||||
{
|
||||
var sourceDir = FindSourceFixturesDirectory();
|
||||
if (sourceDir is not null)
|
||||
{
|
||||
FixturesPath = sourceDir;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get sample SecDB YAML content for main repository.
|
||||
/// </summary>
|
||||
public static string GetSampleSecDbMain()
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, "main.yaml");
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
// Return inline fixture if file doesn't exist
|
||||
return SampleSecDbMainContent;
|
||||
}
|
||||
return File.ReadAllText(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get sample SecDB YAML content for community repository.
|
||||
/// </summary>
|
||||
public static string GetSampleSecDbCommunity()
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, "community.yaml");
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return SampleSecDbCommunityContent;
|
||||
}
|
||||
return File.ReadAllText(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Get a fixture file as a stream.
|
||||
/// </summary>
|
||||
public static Stream GetFixtureStream(string name)
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, name);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
throw new FileNotFoundException($"Fixture not found: {path}");
|
||||
}
|
||||
return File.OpenRead(path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if a fixture exists.
|
||||
/// </summary>
|
||||
public static bool FixtureExists(string name)
|
||||
{
|
||||
var path = Path.Combine(FixturesPath, name);
|
||||
return File.Exists(path);
|
||||
}
|
||||
|
||||
private static string? FindSourceFixturesDirectory()
|
||||
{
|
||||
var dir = Directory.GetCurrentDirectory();
|
||||
while (dir is not null)
|
||||
{
|
||||
var candidate = Path.Combine(dir, "src", "BinaryIndex", "__Tests",
|
||||
"StellaOps.BinaryIndex.GroundTruth.SecDb.Tests", "Fixtures");
|
||||
if (Directory.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
dir = Directory.GetParent(dir)?.FullName;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inline sample SecDB main.yaml content for deterministic testing.
|
||||
/// Based on Alpine SecDB format.
|
||||
/// </summary>
|
||||
private const string SampleSecDbMainContent = """
|
||||
distroversion: v3.19
|
||||
reponame: main
|
||||
urlprefix: https://dl-cdn.alpinelinux.org/alpine/v3.19/main
|
||||
packages:
|
||||
- pkg: curl
|
||||
secfixes:
|
||||
8.5.0-r0:
|
||||
- CVE-2023-46218 Improper validation of HTTP headers
|
||||
- CVE-2023-46219 Double free in async URL resolver
|
||||
8.4.0-r0:
|
||||
- CVE-2023-38545 SOCKS5 heap buffer overflow
|
||||
8.1.2-r0:
|
||||
- CVE-2023-27535 FTP injection vulnerability
|
||||
- pkg: openssl
|
||||
secfixes:
|
||||
3.1.4-r3:
|
||||
- CVE-2024-0727 PKCS12 decoding crash
|
||||
3.1.4-r0:
|
||||
- CVE-2023-5678 Denial of service
|
||||
3.1.2-r0:
|
||||
- CVE-2023-3817 Excessive time checking DH parameters
|
||||
- pkg: linux-lts
|
||||
secfixes:
|
||||
6.1.67-r0:
|
||||
- CVE-2023-6817 Use-after-free in netfilter
|
||||
- CVE-2023-6606 Out-of-bounds read in SMB
|
||||
6.1.64-r0:
|
||||
- CVE-2023-5717 User-mode root exploit via perf
|
||||
""";
|
||||
|
||||
/// <summary>
|
||||
/// Inline sample SecDB community.yaml content.
|
||||
/// </summary>
|
||||
private const string SampleSecDbCommunityContent = """
|
||||
distroversion: v3.19
|
||||
reponame: community
|
||||
urlprefix: https://dl-cdn.alpinelinux.org/alpine/v3.19/community
|
||||
packages:
|
||||
- pkg: go
|
||||
secfixes:
|
||||
1.21.5-r0:
|
||||
- CVE-2023-45283 Path traversal on Windows
|
||||
- CVE-2023-45284 Runtime panic in crypto/tls
|
||||
1.21.4-r0:
|
||||
- CVE-2023-44487 HTTP/2 rapid reset attack
|
||||
- pkg: nodejs
|
||||
secfixes:
|
||||
20.10.0-r0:
|
||||
- CVE-2023-46809 Permissions policy bypass
|
||||
20.9.0-r0:
|
||||
- CVE-2023-38552 Integrity bypass via TLS/HTTPS
|
||||
- pkg: chromium
|
||||
secfixes:
|
||||
120.0.6099.71-r0:
|
||||
- CVE-2023-6702 Type confusion in V8
|
||||
119.0.6045.199-r0:
|
||||
- CVE-2023-6345 Integer overflow in Skia
|
||||
- pkg: unfixed-example
|
||||
secfixes:
|
||||
"0":
|
||||
- CVE-2023-99999 Example unfixed vulnerability
|
||||
""";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test fixture constants for SecDB tests.
|
||||
/// </summary>
|
||||
public static class FixtureConstants
|
||||
{
|
||||
// Sample package info
|
||||
public const string SamplePackageCurl = "curl";
|
||||
public const string SamplePackageOpenssl = "openssl";
|
||||
public const string SamplePackageGo = "go";
|
||||
public const string SamplePackageNodejs = "nodejs";
|
||||
|
||||
// Sample branches
|
||||
public const string SampleBranchV319 = "v3.19";
|
||||
public const string SampleBranchEdge = "edge";
|
||||
|
||||
// Sample repositories
|
||||
public const string SampleRepoMain = "main";
|
||||
public const string SampleRepoCommunity = "community";
|
||||
|
||||
// Expected CVE counts
|
||||
public const int ExpectedCurlCveCount = 4;
|
||||
public const int ExpectedOpensslCveCount = 3;
|
||||
|
||||
// Sample CVEs
|
||||
public const string SampleCveCurl = "CVE-2023-46218";
|
||||
public const string SampleCveOpenssl = "CVE-2024-0727";
|
||||
public const string SampleCveUnfixed = "CVE-2023-99999";
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.BinaryIndex.GroundTruth.SecDb.Configuration;
|
||||
using StellaOps.BinaryIndex.GroundTruth.SecDb.Tests.Fixtures;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.SecDb.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for SecDb connector.
|
||||
/// These tests require network access to gitlab.alpinelinux.org.
|
||||
/// Skip in CI by setting SKIP_INTEGRATION_TESTS=true.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public class SecDbConnectorIntegrationTests : IAsyncLifetime
|
||||
{
|
||||
private ServiceProvider? _services;
|
||||
private readonly bool _skipTests;
|
||||
|
||||
public SecDbConnectorIntegrationTests()
|
||||
{
|
||||
_skipTests = Environment.GetEnvironmentVariable("SKIP_INTEGRATION_TESTS")?.ToLowerInvariant() == "true"
|
||||
|| Environment.GetEnvironmentVariable("CI")?.ToLowerInvariant() == "true";
|
||||
}
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
if (_skipTests)
|
||||
return Task.CompletedTask;
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddConsole().SetMinimumLevel(LogLevel.Debug));
|
||||
services.AddSecDbConnector(opts =>
|
||||
{
|
||||
opts.Branches = ["v3.19"];
|
||||
opts.Repositories = ["main"];
|
||||
opts.TimeoutSeconds = 120;
|
||||
opts.FetchAports = false; // Don't fetch aports for integration tests
|
||||
});
|
||||
|
||||
_services = services.BuildServiceProvider();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_services?.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecDbConnector_CanTestConnectivity()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<SecDbConnector>();
|
||||
|
||||
// Act
|
||||
var result = await connector.TestConnectivityAsync();
|
||||
|
||||
// Assert
|
||||
result.IsConnected.Should().BeTrue("Should be able to connect to Alpine GitLab");
|
||||
result.Latency.Should().BeLessThan(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecDbConnector_CanGetMetadata()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<SecDbConnector>();
|
||||
|
||||
// Act
|
||||
var metadata = await connector.GetMetadataAsync();
|
||||
|
||||
// Assert
|
||||
metadata.SourceId.Should().Be("secdb-alpine");
|
||||
metadata.DisplayName.Should().Contain("Alpine");
|
||||
metadata.BaseUrl.Should().Contain("gitlab.alpinelinux.org");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SecDbConnector_HasCorrectProperties()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<SecDbConnector>();
|
||||
|
||||
// Assert
|
||||
connector.SourceId.Should().Be("secdb-alpine");
|
||||
connector.DisplayName.Should().Contain("SecDB");
|
||||
connector.SupportedDistros.Should().Contain("alpine");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SecDbConnector_FetchAndGetVulnerabilities_ReturnsData()
|
||||
{
|
||||
Skip.If(_skipTests, "Integration tests skipped");
|
||||
|
||||
// Arrange
|
||||
var connector = _services!.GetRequiredService<SecDbConnector>();
|
||||
|
||||
// First fetch the data
|
||||
await connector.FetchAsync(_services!, CancellationToken.None);
|
||||
|
||||
// Act - get vulnerabilities for a well-known package
|
||||
var vulnerabilities = await connector.GetVulnerabilitiesForPackageAsync("curl");
|
||||
|
||||
// Assert
|
||||
vulnerabilities.Should().NotBeEmpty("curl should have known vulnerabilities");
|
||||
vulnerabilities.Should().OnlyContain(v => v.CveId.StartsWith("CVE-"));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides Skip functionality for xUnit when condition is true.
|
||||
/// </summary>
|
||||
public static class Skip
|
||||
{
|
||||
public static void If(bool condition, string reason)
|
||||
{
|
||||
if (condition)
|
||||
{
|
||||
throw new SkipException(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception to skip a test.
|
||||
/// </summary>
|
||||
public class SkipException : Exception
|
||||
{
|
||||
public SkipException(string reason) : base(reason) { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test meter factory for diagnostics.
|
||||
/// </summary>
|
||||
internal sealed class TestMeterFactory : System.Diagnostics.Metrics.IMeterFactory
|
||||
{
|
||||
public System.Diagnostics.Metrics.Meter Create(System.Diagnostics.Metrics.MeterOptions options)
|
||||
=> new(options.Name, options.Version);
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
using FluentAssertions;
|
||||
using StellaOps.BinaryIndex.GroundTruth.SecDb.Internal;
|
||||
using StellaOps.BinaryIndex.GroundTruth.SecDb.Tests.Fixtures;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.GroundTruth.SecDb.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for SecDbParser using deterministic fixtures.
|
||||
/// </summary>
|
||||
public class SecDbParserTests
|
||||
{
|
||||
private readonly SecDbParser _parser = new();
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleMainYaml_ParsesDistroVersion()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbMain();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
result.DistroVersion.Should().Be(FixtureConstants.SampleBranchV319);
|
||||
result.RepoName.Should().Be(FixtureConstants.SampleRepoMain);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleMainYaml_ExtractsPackages()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbMain();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
result.Packages.Should().HaveCountGreaterThanOrEqualTo(3);
|
||||
result.Packages.Should().Contain(p => p.Name == FixtureConstants.SamplePackageCurl);
|
||||
result.Packages.Should().Contain(p => p.Name == FixtureConstants.SamplePackageOpenssl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleMainYaml_ExtractsCurlVulnerabilities()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbMain();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
var curl = result.Packages.First(p => p.Name == FixtureConstants.SamplePackageCurl);
|
||||
curl.Vulnerabilities.Should().HaveCount(FixtureConstants.ExpectedCurlCveCount);
|
||||
curl.Vulnerabilities.Should().Contain(v => v.CveId == FixtureConstants.SampleCveCurl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleMainYaml_ExtractsFixedVersions()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbMain();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
var curl = result.Packages.First(p => p.Name == FixtureConstants.SamplePackageCurl);
|
||||
var cve = curl.Vulnerabilities.First(v => v.CveId == FixtureConstants.SampleCveCurl);
|
||||
cve.FixedInVersion.Should().Be("8.5.0-r0");
|
||||
cve.IsUnfixed.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleCommunityYaml_ParsesCommunityPackages()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbCommunity();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoCommunity);
|
||||
|
||||
// Assert
|
||||
result.Packages.Should().Contain(p => p.Name == FixtureConstants.SamplePackageGo);
|
||||
result.Packages.Should().Contain(p => p.Name == FixtureConstants.SamplePackageNodejs);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleCommunityYaml_DetectsUnfixedVulnerabilities()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbCommunity();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoCommunity);
|
||||
|
||||
// Assert
|
||||
var unfixedPkg = result.Packages.First(p => p.Name == "unfixed-example");
|
||||
var unfixedCve = unfixedPkg.Vulnerabilities.First(v => v.CveId == FixtureConstants.SampleCveUnfixed);
|
||||
unfixedCve.FixedInVersion.Should().Be("0");
|
||||
unfixedCve.IsUnfixed.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SampleMainYaml_CalculatesTotalVulnerabilityCount()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbMain();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
result.VulnerabilityCount.Should().BeGreaterThan(0);
|
||||
result.VulnerabilityCount.Should().Be(result.Packages.Sum(p => p.Vulnerabilities.Count));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_CveWithDescription_ExtractsDescription()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbMain();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
var curl = result.Packages.First(p => p.Name == FixtureConstants.SamplePackageCurl);
|
||||
var cve = curl.Vulnerabilities.First(v => v.CveId == FixtureConstants.SampleCveCurl);
|
||||
cve.Description.Should().Contain("HTTP headers");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_EmptyContent_ThrowsFormatException()
|
||||
{
|
||||
// Act
|
||||
var act = () => _parser.Parse("", FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
act.Should().Throw<FormatException>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_InvalidYaml_ThrowsFormatException()
|
||||
{
|
||||
// Arrange
|
||||
var content = "this is not valid yaml: [incomplete";
|
||||
|
||||
// Act
|
||||
var act = () => _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
act.Should().Throw<FormatException>();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_EmptyPackagesArray_ReturnsEmptyPackageList()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
distroversion: v3.19
|
||||
reponame: main
|
||||
packages: []
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
result.Packages.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_PackageWithNoSecfixes_ReturnsEmptyVulnerabilities()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
distroversion: v3.19
|
||||
reponame: main
|
||||
packages:
|
||||
- pkg: no-vulns-pkg
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
result.Packages.Should().HaveCount(1);
|
||||
result.Packages[0].Vulnerabilities.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_NonCveEntry_SkipsNonCveIdentifiers()
|
||||
{
|
||||
// Arrange - Alpine secdb sometimes has XSA-xxx or other identifiers
|
||||
var content = """
|
||||
distroversion: v3.19
|
||||
reponame: main
|
||||
packages:
|
||||
- pkg: xen
|
||||
secfixes:
|
||||
4.18.0-r1:
|
||||
- XSA-445 Not a CVE
|
||||
- CVE-2023-12345 Actual CVE
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
var xen = result.Packages.First();
|
||||
xen.Vulnerabilities.Should().HaveCount(1);
|
||||
xen.Vulnerabilities[0].CveId.Should().Be("CVE-2023-12345");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_CveIdNormalization_ConvertsToUppercase()
|
||||
{
|
||||
// Arrange
|
||||
var content = """
|
||||
distroversion: v3.19
|
||||
reponame: main
|
||||
packages:
|
||||
- pkg: test
|
||||
secfixes:
|
||||
1.0-r0:
|
||||
- cve-2023-12345 lowercase
|
||||
""";
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
var pkg = result.Packages.First();
|
||||
pkg.Vulnerabilities[0].CveId.Should().Be("CVE-2023-12345");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_MultipleCvesInSameVersion_ParsesAll()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbMain();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
var linuxLts = result.Packages.First(p => p.Name == "linux-lts");
|
||||
var version6167 = linuxLts.Vulnerabilities.Where(v => v.FixedInVersion == "6.1.67-r0").ToList();
|
||||
version6167.Should().HaveCount(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_SetsBranchAndRepository()
|
||||
{
|
||||
// Arrange
|
||||
var content = FixtureProvider.GetSampleSecDbMain();
|
||||
|
||||
// Act
|
||||
var result = _parser.Parse(content, FixtureConstants.SampleBranchV319, FixtureConstants.SampleRepoMain);
|
||||
|
||||
// Assert
|
||||
result.Branch.Should().Be(FixtureConstants.SampleBranchV319);
|
||||
result.Repository.Should().Be(FixtureConstants.SampleRepoMain);
|
||||
|
||||
foreach (var pkg in result.Packages)
|
||||
{
|
||||
pkg.Branch.Should().Be(FixtureConstants.SampleBranchV319);
|
||||
pkg.Repository.Should().Be(FixtureConstants.SampleRepoMain);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.BinaryIndex.GroundTruth.SecDb\StellaOps.BinaryIndex.GroundTruth.SecDb.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="Fixtures\**\*">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,195 @@
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using StellaOps.BinaryIndex.Validation.Abstractions;
|
||||
using StellaOps.BinaryIndex.Validation.Attestation;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.Validation.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for ValidationRunAttestor.
|
||||
/// </summary>
|
||||
public class ValidationRunAttestorTests
|
||||
{
|
||||
private readonly ValidationRunAttestor _sut = new();
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateAttestationAsync_ProducesDsseEnvelope()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var json = await _sut.GenerateAttestationAsync(run);
|
||||
|
||||
// Assert
|
||||
var envelope = JsonDocument.Parse(json);
|
||||
envelope.RootElement.GetProperty("payloadType").GetString()
|
||||
.Should().Be("application/vnd.in-toto+json");
|
||||
envelope.RootElement.GetProperty("payload").GetString()
|
||||
.Should().NotBeNullOrEmpty();
|
||||
envelope.RootElement.GetProperty("signatures").GetArrayLength()
|
||||
.Should().Be(0); // No signer provided
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateAttestationAsync_PayloadContainsStatement()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var json = await _sut.GenerateAttestationAsync(run);
|
||||
var envelope = JsonDocument.Parse(json);
|
||||
var payloadBase64 = envelope.RootElement.GetProperty("payload").GetString()!;
|
||||
var payloadJson = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(payloadBase64));
|
||||
var statement = JsonDocument.Parse(payloadJson);
|
||||
|
||||
// Assert
|
||||
statement.RootElement.GetProperty("_type").GetString()
|
||||
.Should().Be("https://in-toto.io/Statement/v1");
|
||||
statement.RootElement.GetProperty("predicateType").GetString()
|
||||
.Should().Be("https://stella-ops.org/predicates/validation-run/v1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateAttestationAsync_SubjectIncludesRunId()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var json = await _sut.GenerateAttestationAsync(run);
|
||||
var envelope = JsonDocument.Parse(json);
|
||||
var payloadBase64 = envelope.RootElement.GetProperty("payload").GetString()!;
|
||||
var payloadJson = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(payloadBase64));
|
||||
var statement = JsonDocument.Parse(payloadJson);
|
||||
|
||||
// Assert
|
||||
var subject = statement.RootElement.GetProperty("subject")[0];
|
||||
subject.GetProperty("name").GetString()
|
||||
.Should().Contain(run.Id.ToString());
|
||||
subject.GetProperty("digest").GetProperty("sha256").GetString()
|
||||
.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateAttestationAsync_PredicateContainsMetrics()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var json = await _sut.GenerateAttestationAsync(run);
|
||||
var envelope = JsonDocument.Parse(json);
|
||||
var payloadBase64 = envelope.RootElement.GetProperty("payload").GetString()!;
|
||||
var payloadJson = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(payloadBase64));
|
||||
var statement = JsonDocument.Parse(payloadJson);
|
||||
var predicate = statement.RootElement.GetProperty("predicate");
|
||||
|
||||
// Assert
|
||||
predicate.GetProperty("runId").GetString().Should().Be(run.Id.ToString());
|
||||
predicate.GetProperty("runName").GetString().Should().Be(run.Config.Name);
|
||||
|
||||
var metrics = predicate.GetProperty("metrics");
|
||||
metrics.GetProperty("truePositives").GetInt32().Should().Be(run.Metrics!.TruePositives);
|
||||
metrics.GetProperty("falsePositives").GetInt32().Should().Be(run.Metrics.FalsePositives);
|
||||
metrics.GetProperty("f1Score").GetDouble().Should().BeApproximately(run.Metrics.F1Score, 0.001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateAttestationAsync_PredicateContainsConfiguration()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var json = await _sut.GenerateAttestationAsync(run);
|
||||
var envelope = JsonDocument.Parse(json);
|
||||
var payloadBase64 = envelope.RootElement.GetProperty("payload").GetString()!;
|
||||
var payloadJson = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(payloadBase64));
|
||||
var statement = JsonDocument.Parse(payloadJson);
|
||||
var config = statement.RootElement.GetProperty("predicate").GetProperty("configuration");
|
||||
|
||||
// Assert
|
||||
config.GetProperty("matcherType").GetString().Should().Be("SemanticDiff");
|
||||
config.GetProperty("minMatchScore").GetDouble().Should().Be(0.5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateAttestationAsync_IncludesCorpusInfo()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var json = await _sut.GenerateAttestationAsync(run);
|
||||
var envelope = JsonDocument.Parse(json);
|
||||
var payloadBase64 = envelope.RootElement.GetProperty("payload").GetString()!;
|
||||
var payloadJson = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(payloadBase64));
|
||||
var statement = JsonDocument.Parse(payloadJson);
|
||||
var corpus = statement.RootElement.GetProperty("predicate").GetProperty("corpus");
|
||||
|
||||
// Assert
|
||||
corpus.GetProperty("snapshotId").GetString().Should().Be(run.CorpusSnapshotId);
|
||||
corpus.GetProperty("pairsEvaluated").GetInt32().Should().Be(run.Metrics!.TotalPairs);
|
||||
corpus.GetProperty("functionsEvaluated").GetInt32().Should().Be(run.Metrics.TotalFunctions);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateAttestationAsync_ThrowsForPendingRun()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun() with { Status = ValidationRunStatus.Pending };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _sut.GenerateAttestationAsync(run);
|
||||
await act.Should().ThrowAsync<InvalidOperationException>()
|
||||
.WithMessage("*completed*");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GenerateAttestationAsync_ThrowsForRunWithoutMetrics()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun() with { Metrics = null };
|
||||
|
||||
// Act & Assert
|
||||
var act = () => _sut.GenerateAttestationAsync(run);
|
||||
await act.Should().ThrowAsync<InvalidOperationException>()
|
||||
.WithMessage("*metrics*");
|
||||
}
|
||||
|
||||
private static ValidationRun CreateCompletedRun()
|
||||
{
|
||||
return new ValidationRun
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Config = new ValidationConfig
|
||||
{
|
||||
Name = "Attestation Test Run",
|
||||
Matcher = new MatcherConfig { Type = MatcherType.SemanticDiff },
|
||||
MinMatchScore = 0.5,
|
||||
MaxFalsePositiveRate = 0.05,
|
||||
MaxFalseNegativeRate = 0.10
|
||||
},
|
||||
Status = ValidationRunStatus.Completed,
|
||||
CreatedAt = DateTimeOffset.UtcNow.AddHours(-1),
|
||||
StartedAt = DateTimeOffset.UtcNow.AddMinutes(-30),
|
||||
CompletedAt = DateTimeOffset.UtcNow,
|
||||
CorpusSnapshotId = "corpus-16-1705680000",
|
||||
MatcherVersion = "1.0.0",
|
||||
Metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 16,
|
||||
TotalFunctions = 1024,
|
||||
TruePositives = 920,
|
||||
FalsePositives = 30,
|
||||
TrueNegatives = 50,
|
||||
FalseNegatives = 24,
|
||||
AverageMatchScore = 0.92,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
using FluentAssertions;
|
||||
using StellaOps.BinaryIndex.Validation;
|
||||
using StellaOps.BinaryIndex.Validation.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.Validation.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for MetricsCalculator.
|
||||
/// </summary>
|
||||
public class MetricsCalculatorTests
|
||||
{
|
||||
private readonly MetricsCalculator _sut = new();
|
||||
|
||||
[Fact]
|
||||
public void Calculate_WithEmptyResults_ReturnsZeroMetrics()
|
||||
{
|
||||
// Arrange
|
||||
var results = new List<MatchResult>();
|
||||
|
||||
// Act
|
||||
var metrics = _sut.Calculate(results);
|
||||
|
||||
// Assert
|
||||
metrics.TotalFunctions.Should().Be(0);
|
||||
metrics.TotalPairs.Should().Be(0);
|
||||
metrics.TruePositives.Should().Be(0);
|
||||
metrics.FalsePositives.Should().Be(0);
|
||||
metrics.TrueNegatives.Should().Be(0);
|
||||
metrics.FalseNegatives.Should().Be(0);
|
||||
metrics.MatchRate.Should().Be(0);
|
||||
metrics.Precision.Should().Be(0);
|
||||
metrics.Recall.Should().Be(0);
|
||||
metrics.F1Score.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Calculate_WithAllTruePositives_ReturnsPerfectMetrics()
|
||||
{
|
||||
// Arrange
|
||||
var results = CreateMatchResults(
|
||||
truePositives: 10,
|
||||
falsePositives: 0,
|
||||
trueNegatives: 0,
|
||||
falseNegatives: 0);
|
||||
|
||||
// Act
|
||||
var metrics = _sut.Calculate(results);
|
||||
|
||||
// Assert
|
||||
metrics.TotalFunctions.Should().Be(10);
|
||||
metrics.TruePositives.Should().Be(10);
|
||||
metrics.MatchRate.Should().Be(1.0);
|
||||
metrics.Precision.Should().Be(1.0);
|
||||
metrics.Recall.Should().Be(1.0);
|
||||
metrics.F1Score.Should().Be(1.0);
|
||||
metrics.Accuracy.Should().Be(1.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Calculate_WithMixedResults_CalculatesCorrectMetrics()
|
||||
{
|
||||
// Arrange
|
||||
var results = CreateMatchResults(
|
||||
truePositives: 80,
|
||||
falsePositives: 10,
|
||||
trueNegatives: 5,
|
||||
falseNegatives: 5);
|
||||
|
||||
// Act
|
||||
var metrics = _sut.Calculate(results);
|
||||
|
||||
// Assert
|
||||
metrics.TotalFunctions.Should().Be(100);
|
||||
metrics.TruePositives.Should().Be(80);
|
||||
metrics.FalsePositives.Should().Be(10);
|
||||
metrics.TrueNegatives.Should().Be(5);
|
||||
metrics.FalseNegatives.Should().Be(5);
|
||||
|
||||
// Precision = TP / (TP + FP) = 80 / 90 ≈ 0.889
|
||||
metrics.Precision.Should().BeApproximately(0.889, 0.001);
|
||||
|
||||
// Recall = TP / (TP + FN) = 80 / 85 ≈ 0.941
|
||||
metrics.Recall.Should().BeApproximately(0.941, 0.001);
|
||||
|
||||
// F1 = 2 * (P * R) / (P + R)
|
||||
var expectedF1 = 2 * (metrics.Precision * metrics.Recall) / (metrics.Precision + metrics.Recall);
|
||||
metrics.F1Score.Should().BeApproximately(expectedF1, 0.001);
|
||||
|
||||
// Accuracy = (TP + TN) / Total = 85 / 100 = 0.85
|
||||
metrics.Accuracy.Should().Be(0.85);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Calculate_WithOnlyFalseNegatives_ReturnsZeroRecall()
|
||||
{
|
||||
// Arrange
|
||||
var results = CreateMatchResults(
|
||||
truePositives: 0,
|
||||
falsePositives: 0,
|
||||
trueNegatives: 0,
|
||||
falseNegatives: 10);
|
||||
|
||||
// Act
|
||||
var metrics = _sut.Calculate(results);
|
||||
|
||||
// Assert
|
||||
metrics.Recall.Should().Be(0);
|
||||
metrics.Precision.Should().Be(0); // No positives predicted
|
||||
metrics.F1Score.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Calculate_TracksMultiplePairs()
|
||||
{
|
||||
// Arrange
|
||||
var pairId1 = Guid.NewGuid();
|
||||
var pairId2 = Guid.NewGuid();
|
||||
var results = new List<MatchResult>
|
||||
{
|
||||
CreateSingleResult(MatchOutcome.TruePositive, pairId1),
|
||||
CreateSingleResult(MatchOutcome.TruePositive, pairId1),
|
||||
CreateSingleResult(MatchOutcome.FalseNegative, pairId2),
|
||||
CreateSingleResult(MatchOutcome.TruePositive, pairId2)
|
||||
};
|
||||
|
||||
// Act
|
||||
var metrics = _sut.Calculate(results);
|
||||
|
||||
// Assert
|
||||
metrics.TotalPairs.Should().Be(2);
|
||||
metrics.TotalFunctions.Should().Be(4);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0.5, 0.5, 0.5, 0.5)]
|
||||
[InlineData(0.9, 0.9, 0.9, 0.9)]
|
||||
[InlineData(1.0, 0.5, 0.667, 0.5)]
|
||||
public void Calculate_MatchScoreStatistics_CalculatedCorrectly(
|
||||
double score1, double score2, double expectedAverage, double expectedMedian)
|
||||
{
|
||||
// Arrange
|
||||
var results = new List<MatchResult>
|
||||
{
|
||||
CreateSingleResultWithScore(MatchOutcome.TruePositive, score1),
|
||||
CreateSingleResultWithScore(MatchOutcome.TruePositive, score2)
|
||||
};
|
||||
|
||||
// Act
|
||||
var metrics = _sut.Calculate(results);
|
||||
|
||||
// Assert
|
||||
metrics.AverageMatchScore.Should().BeApproximately(expectedAverage, 0.01);
|
||||
metrics.MedianMatchScore.Should().BeApproximately(expectedMedian, 0.01);
|
||||
}
|
||||
|
||||
private static List<MatchResult> CreateMatchResults(
|
||||
int truePositives, int falsePositives, int trueNegatives, int falseNegatives)
|
||||
{
|
||||
var results = new List<MatchResult>();
|
||||
var pairId = Guid.NewGuid();
|
||||
|
||||
for (var i = 0; i < truePositives; i++)
|
||||
results.Add(CreateSingleResult(MatchOutcome.TruePositive, pairId));
|
||||
|
||||
for (var i = 0; i < falsePositives; i++)
|
||||
results.Add(CreateSingleResult(MatchOutcome.FalsePositive, pairId));
|
||||
|
||||
for (var i = 0; i < trueNegatives; i++)
|
||||
results.Add(CreateSingleResult(MatchOutcome.TrueNegative, pairId));
|
||||
|
||||
for (var i = 0; i < falseNegatives; i++)
|
||||
results.Add(CreateSingleResult(MatchOutcome.FalseNegative, pairId));
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static MatchResult CreateSingleResult(MatchOutcome outcome, Guid pairId)
|
||||
{
|
||||
return new MatchResult
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
RunId = Guid.NewGuid(),
|
||||
SecurityPairId = pairId,
|
||||
SourceFunction = CreateFunctionIdentifier("source_func"),
|
||||
ExpectedTarget = CreateFunctionIdentifier("target_func"),
|
||||
ActualTarget = outcome == MatchOutcome.TruePositive ? CreateFunctionIdentifier("target_func") : null,
|
||||
Outcome = outcome,
|
||||
MatchScore = outcome == MatchOutcome.TruePositive ? 0.95 : null
|
||||
};
|
||||
}
|
||||
|
||||
private static MatchResult CreateSingleResultWithScore(MatchOutcome outcome, double score)
|
||||
{
|
||||
return new MatchResult
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
RunId = Guid.NewGuid(),
|
||||
SecurityPairId = Guid.NewGuid(),
|
||||
SourceFunction = CreateFunctionIdentifier("source_func"),
|
||||
ExpectedTarget = CreateFunctionIdentifier("target_func"),
|
||||
ActualTarget = CreateFunctionIdentifier("target_func"),
|
||||
Outcome = outcome,
|
||||
MatchScore = score
|
||||
};
|
||||
}
|
||||
|
||||
private static FunctionIdentifier CreateFunctionIdentifier(string name)
|
||||
{
|
||||
return new FunctionIdentifier
|
||||
{
|
||||
Name = name,
|
||||
Address = 0x1000,
|
||||
BuildId = "abc123",
|
||||
BinaryName = "test.so"
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
using FluentAssertions;
|
||||
using NSubstitute;
|
||||
using StellaOps.BinaryIndex.Validation;
|
||||
using StellaOps.BinaryIndex.Validation.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.Validation.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for MismatchAnalyzer.
|
||||
/// </summary>
|
||||
public class MismatchAnalyzerTests
|
||||
{
|
||||
private readonly IMismatchCauseInferrer _causeInferrer;
|
||||
private readonly MismatchAnalyzer _sut;
|
||||
|
||||
public MismatchAnalyzerTests()
|
||||
{
|
||||
_causeInferrer = Substitute.For<IMismatchCauseInferrer>();
|
||||
_sut = new MismatchAnalyzer(_causeInferrer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnalyzeAsync_WithEmptyList_ReturnsEmptyBuckets()
|
||||
{
|
||||
// Arrange
|
||||
var mismatches = new List<MatchResult>();
|
||||
|
||||
// Act
|
||||
var analysis = await _sut.AnalyzeAsync(mismatches, maxExamplesPerBucket: 5);
|
||||
|
||||
// Assert
|
||||
analysis.Buckets.Should().BeEmpty();
|
||||
analysis.TotalMismatches.Should().Be(0);
|
||||
analysis.DominantCause.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnalyzeAsync_GroupsMismatchesByCause()
|
||||
{
|
||||
// Arrange
|
||||
var mismatches = new List<MatchResult>
|
||||
{
|
||||
CreateMismatch("func1@@GLIBC_2.17"),
|
||||
CreateMismatch("func2@@GLIBC_2.34"),
|
||||
CreateMismatch("small_func", size: 20)
|
||||
};
|
||||
|
||||
_causeInferrer.InferCauseAsync(Arg.Any<MatchResult>(), Arg.Any<CancellationToken>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
var mismatch = callInfo.Arg<MatchResult>();
|
||||
if (mismatch.SourceFunction.Name.Contains("@@"))
|
||||
return (MismatchCause.SymbolVersioning, 0.9);
|
||||
return (MismatchCause.Inlining, 0.6);
|
||||
});
|
||||
|
||||
// Act
|
||||
var analysis = await _sut.AnalyzeAsync(mismatches, maxExamplesPerBucket: 5);
|
||||
|
||||
// Assert
|
||||
analysis.Buckets.Should().HaveCount(2);
|
||||
analysis.TotalMismatches.Should().Be(3);
|
||||
analysis.Buckets[MismatchCause.SymbolVersioning].Count.Should().Be(2);
|
||||
analysis.Buckets[MismatchCause.Inlining].Count.Should().Be(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnalyzeAsync_DominantCause_IsHighestCount()
|
||||
{
|
||||
// Arrange
|
||||
var mismatches = Enumerable.Range(0, 10)
|
||||
.Select(i => CreateMismatch($"func_{i}"))
|
||||
.ToList();
|
||||
|
||||
_causeInferrer.InferCauseAsync(Arg.Any<MatchResult>(), Arg.Any<CancellationToken>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
var index = int.Parse(callInfo.Arg<MatchResult>().SourceFunction.Name.Split('_')[1]);
|
||||
return index < 7
|
||||
? (MismatchCause.OptimizationLevel, 0.8)
|
||||
: (MismatchCause.CompilerVersion, 0.7);
|
||||
});
|
||||
|
||||
// Act
|
||||
var analysis = await _sut.AnalyzeAsync(mismatches, maxExamplesPerBucket: 3);
|
||||
|
||||
// Assert
|
||||
analysis.DominantCause.Should().Be(MismatchCause.OptimizationLevel);
|
||||
analysis.Buckets[MismatchCause.OptimizationLevel].Count.Should().Be(7);
|
||||
analysis.Buckets[MismatchCause.CompilerVersion].Count.Should().Be(3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnalyzeAsync_LimitsExamplesPerBucket()
|
||||
{
|
||||
// Arrange
|
||||
var mismatches = Enumerable.Range(0, 20)
|
||||
.Select(i => CreateMismatch($"func_{i}"))
|
||||
.ToList();
|
||||
|
||||
_causeInferrer.InferCauseAsync(Arg.Any<MatchResult>(), Arg.Any<CancellationToken>())
|
||||
.Returns((MismatchCause.Unknown, 0.5));
|
||||
|
||||
// Act
|
||||
var analysis = await _sut.AnalyzeAsync(mismatches, maxExamplesPerBucket: 5);
|
||||
|
||||
// Assert
|
||||
analysis.Buckets[MismatchCause.Unknown].Examples.Should().HaveCount(5);
|
||||
analysis.Buckets[MismatchCause.Unknown].Count.Should().Be(20);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AnalyzeAsync_CalculatesPercentages()
|
||||
{
|
||||
// Arrange
|
||||
var mismatches = Enumerable.Range(0, 100)
|
||||
.Select(i => CreateMismatch($"func_{i}"))
|
||||
.ToList();
|
||||
|
||||
_causeInferrer.InferCauseAsync(Arg.Any<MatchResult>(), Arg.Any<CancellationToken>())
|
||||
.Returns(callInfo =>
|
||||
{
|
||||
var index = int.Parse(callInfo.Arg<MatchResult>().SourceFunction.Name.Split('_')[1]);
|
||||
return index < 60
|
||||
? (MismatchCause.Inlining, 0.8)
|
||||
: (MismatchCause.LinkTimeOptimization, 0.7);
|
||||
});
|
||||
|
||||
// Act
|
||||
var analysis = await _sut.AnalyzeAsync(mismatches, maxExamplesPerBucket: 5);
|
||||
|
||||
// Assert
|
||||
analysis.Buckets[MismatchCause.Inlining].Percentage.Should().BeApproximately(60, 0.1);
|
||||
analysis.Buckets[MismatchCause.LinkTimeOptimization].Percentage.Should().BeApproximately(40, 0.1);
|
||||
}
|
||||
|
||||
private static MatchResult CreateMismatch(string functionName, ulong? size = null)
|
||||
{
|
||||
return new MatchResult
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
RunId = Guid.NewGuid(),
|
||||
SecurityPairId = Guid.NewGuid(),
|
||||
SourceFunction = new FunctionIdentifier
|
||||
{
|
||||
Name = functionName,
|
||||
Address = 0x1000,
|
||||
Size = size,
|
||||
BuildId = "abc123",
|
||||
BinaryName = "test.so"
|
||||
},
|
||||
ExpectedTarget = new FunctionIdentifier
|
||||
{
|
||||
Name = functionName.Replace("@@GLIBC_2.17", "").Replace("@@GLIBC_2.34", ""),
|
||||
Address = 0x2000,
|
||||
BuildId = "def456",
|
||||
BinaryName = "test.so"
|
||||
},
|
||||
Outcome = MatchOutcome.FalseNegative
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests for HeuristicMismatchCauseInferrer.
|
||||
/// </summary>
|
||||
public class HeuristicMismatchCauseInferrerTests
|
||||
{
|
||||
private readonly HeuristicMismatchCauseInferrer _sut = new();
|
||||
|
||||
[Theory]
|
||||
[InlineData("printf@@GLIBC_2.17", MismatchCause.SymbolVersioning)]
|
||||
[InlineData("malloc@@GLIBC_2.34", MismatchCause.SymbolVersioning)]
|
||||
public async Task InferCauseAsync_SymbolVersioning_DetectedCorrectly(string name, MismatchCause expected)
|
||||
{
|
||||
// Arrange
|
||||
var mismatch = CreateMismatch(name);
|
||||
|
||||
// Act
|
||||
var (cause, confidence) = await _sut.InferCauseAsync(mismatch);
|
||||
|
||||
// Assert
|
||||
cause.Should().Be(expected);
|
||||
confidence.Should().BeGreaterThan(0.8);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("small_func", 20, MismatchCause.Inlining)]
|
||||
[InlineData("tiny_func", 10, MismatchCause.Inlining)]
|
||||
public async Task InferCauseAsync_SmallFunction_InfersInlining(string name, ulong size, MismatchCause expected)
|
||||
{
|
||||
// Arrange
|
||||
var mismatch = CreateMismatch(name, size);
|
||||
|
||||
// Act
|
||||
var (cause, _) = await _sut.InferCauseAsync(mismatch);
|
||||
|
||||
// Assert
|
||||
cause.Should().Be(expected);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("func.cold", MismatchCause.FunctionSplit)]
|
||||
[InlineData("func.isra.0", MismatchCause.FunctionSplit)]
|
||||
[InlineData("func.part.1", MismatchCause.FunctionSplit)]
|
||||
public async Task InferCauseAsync_SplitFunction_DetectedCorrectly(string name, MismatchCause expected)
|
||||
{
|
||||
// Arrange
|
||||
var mismatch = CreateMismatch(name, 500);
|
||||
|
||||
// Act
|
||||
var (cause, confidence) = await _sut.InferCauseAsync(mismatch);
|
||||
|
||||
// Assert
|
||||
cause.Should().Be(expected);
|
||||
confidence.Should().BeGreaterThan(0.7);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("__asan_load8", MismatchCause.SanitizerInstrumentation)]
|
||||
[InlineData("__tsan_write4", MismatchCause.SanitizerInstrumentation)]
|
||||
[InlineData("__ubsan_handle_divrem_overflow", MismatchCause.SanitizerInstrumentation)]
|
||||
public async Task InferCauseAsync_Sanitizer_DetectedCorrectly(string name, MismatchCause expected)
|
||||
{
|
||||
// Arrange
|
||||
var mismatch = CreateMismatch(name, 100);
|
||||
|
||||
// Act
|
||||
var (cause, confidence) = await _sut.InferCauseAsync(mismatch);
|
||||
|
||||
// Assert
|
||||
cause.Should().Be(expected);
|
||||
confidence.Should().BeGreaterThan(0.9);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task InferCauseAsync_UnknownPattern_ReturnsUnknown()
|
||||
{
|
||||
// Arrange
|
||||
var mismatch = CreateMismatch("normal_large_function", 1000);
|
||||
|
||||
// Act
|
||||
var (cause, confidence) = await _sut.InferCauseAsync(mismatch);
|
||||
|
||||
// Assert
|
||||
cause.Should().Be(MismatchCause.Unknown);
|
||||
confidence.Should().BeLessThan(0.5);
|
||||
}
|
||||
|
||||
private static MatchResult CreateMismatch(string name, ulong? size = null)
|
||||
{
|
||||
return new MatchResult
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
RunId = Guid.NewGuid(),
|
||||
SecurityPairId = Guid.NewGuid(),
|
||||
SourceFunction = new FunctionIdentifier
|
||||
{
|
||||
Name = name,
|
||||
Address = 0x1000,
|
||||
Size = size,
|
||||
BuildId = "abc123",
|
||||
BinaryName = "test.so"
|
||||
},
|
||||
ExpectedTarget = new FunctionIdentifier
|
||||
{
|
||||
Name = name,
|
||||
Address = 0x2000,
|
||||
BuildId = "def456",
|
||||
BinaryName = "test.so"
|
||||
},
|
||||
Outcome = MatchOutcome.FalseNegative
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
using FluentAssertions;
|
||||
using StellaOps.BinaryIndex.Validation.Abstractions;
|
||||
using StellaOps.BinaryIndex.Validation.Reports;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.Validation.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for report generators.
|
||||
/// </summary>
|
||||
public class ReportGeneratorTests
|
||||
{
|
||||
private readonly MarkdownReportGenerator _mdGenerator = new();
|
||||
private readonly HtmlReportGenerator _htmlGenerator = new();
|
||||
|
||||
[Fact]
|
||||
public async Task MarkdownGenerator_IncludesRunInfo()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var report = await _mdGenerator.GenerateAsync(run);
|
||||
|
||||
// Assert
|
||||
report.Should().Contain("# Validation Report:");
|
||||
report.Should().Contain(run.Config.Name);
|
||||
report.Should().Contain(run.Id.ToString());
|
||||
report.Should().Contain("Completed");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkdownGenerator_IncludesMetricsTable()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var report = await _mdGenerator.GenerateAsync(run);
|
||||
|
||||
// Assert
|
||||
report.Should().Contain("## Metrics Summary");
|
||||
report.Should().Contain("| Metric | Value |");
|
||||
report.Should().Contain("Match Rate");
|
||||
report.Should().Contain("Precision");
|
||||
report.Should().Contain("Recall");
|
||||
report.Should().Contain("F1 Score");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkdownGenerator_IncludesConfusionMatrix()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var report = await _mdGenerator.GenerateAsync(run);
|
||||
|
||||
// Assert
|
||||
report.Should().Contain("### Confusion Matrix");
|
||||
report.Should().Contain("TP:");
|
||||
report.Should().Contain("FP:");
|
||||
report.Should().Contain("TN:");
|
||||
report.Should().Contain("FN:");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkdownGenerator_WithBaseline_ShowsComparison()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
var baseline = CreateCompletedRun();
|
||||
baseline = baseline with
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Config = baseline.Config with { Name = "Baseline Run" }
|
||||
};
|
||||
|
||||
// Act
|
||||
var report = await _mdGenerator.GenerateAsync(run, baseline);
|
||||
|
||||
// Assert
|
||||
report.Should().Contain("### Comparison with Baseline");
|
||||
report.Should().Contain("Baseline Run");
|
||||
report.Should().Contain("Delta");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkdownGenerator_WithMismatchAnalysis_ShowsBuckets()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
run = run with
|
||||
{
|
||||
MismatchAnalysis = new MismatchAnalysis
|
||||
{
|
||||
Buckets = new Dictionary<MismatchCause, MismatchBucket>
|
||||
{
|
||||
[MismatchCause.Inlining] = new MismatchBucket
|
||||
{
|
||||
Cause = MismatchCause.Inlining,
|
||||
Count = 15,
|
||||
Percentage = 60,
|
||||
Examples = [
|
||||
new MismatchExample
|
||||
{
|
||||
MatchResultId = Guid.NewGuid(),
|
||||
SourceFunction = "small_helper",
|
||||
ExpectedTarget = "small_helper",
|
||||
ActualTarget = null,
|
||||
Explanation = "Function was inlined"
|
||||
}
|
||||
],
|
||||
CommonPatterns = ["Small functions"],
|
||||
SuggestedActions = ["Add inlining normalizer"]
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Act
|
||||
var report = await _mdGenerator.GenerateAsync(run);
|
||||
|
||||
// Assert
|
||||
report.Should().Contain("## Mismatch Analysis");
|
||||
report.Should().Contain("Function Inlining");
|
||||
report.Should().Contain("60");
|
||||
report.Should().Contain("small_helper");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkdownGenerator_WithError_ShowsErrorSection()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
run = run with
|
||||
{
|
||||
Status = ValidationRunStatus.Failed,
|
||||
ErrorMessage = "Connection timeout while loading corpus"
|
||||
};
|
||||
|
||||
// Act
|
||||
var report = await _mdGenerator.GenerateAsync(run);
|
||||
|
||||
// Assert
|
||||
report.Should().Contain("## Error");
|
||||
report.Should().Contain("Connection timeout");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HtmlGenerator_ProducesValidHtml()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var report = await _htmlGenerator.GenerateAsync(run);
|
||||
|
||||
// Assert
|
||||
report.Should().StartWith("<!DOCTYPE html>");
|
||||
report.Should().Contain("<html");
|
||||
report.Should().Contain("</html>");
|
||||
report.Should().Contain("<head>");
|
||||
report.Should().Contain("<body>");
|
||||
report.Should().Contain("<style>");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HtmlGenerator_IncludesTitle()
|
||||
{
|
||||
// Arrange
|
||||
var run = CreateCompletedRun();
|
||||
|
||||
// Act
|
||||
var report = await _htmlGenerator.GenerateAsync(run);
|
||||
|
||||
// Assert
|
||||
report.Should().Contain($"<title>Validation Report: {run.Config.Name}</title>");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HtmlGenerator_Format_ReturnsHtml()
|
||||
{
|
||||
// Assert
|
||||
_htmlGenerator.Format.Should().Be("html");
|
||||
_mdGenerator.Format.Should().Be("markdown");
|
||||
}
|
||||
|
||||
private static ValidationRun CreateCompletedRun()
|
||||
{
|
||||
return new ValidationRun
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Config = new ValidationConfig
|
||||
{
|
||||
Name = "Test Validation Run",
|
||||
Description = "Unit test run",
|
||||
Matcher = new MatcherConfig
|
||||
{
|
||||
Type = MatcherType.SemanticDiff,
|
||||
Options = new Dictionary<string, string> { ["threshold"] = "0.6" }
|
||||
},
|
||||
MinMatchScore = 0.5,
|
||||
MaxFalsePositiveRate = 0.05,
|
||||
MaxFalseNegativeRate = 0.10
|
||||
},
|
||||
Status = ValidationRunStatus.Completed,
|
||||
CreatedAt = DateTimeOffset.UtcNow.AddHours(-1),
|
||||
StartedAt = DateTimeOffset.UtcNow.AddMinutes(-30),
|
||||
CompletedAt = DateTimeOffset.UtcNow,
|
||||
CorpusSnapshotId = "corpus-16-1705680000",
|
||||
MatcherVersion = "1.0.0",
|
||||
Metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 16,
|
||||
TotalFunctions = 1024,
|
||||
TruePositives = 920,
|
||||
FalsePositives = 30,
|
||||
TrueNegatives = 50,
|
||||
FalseNegatives = 24,
|
||||
AverageMatchScore = 0.92,
|
||||
MedianMatchScore = 0.95,
|
||||
P95MatchScore = 0.99,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>
|
||||
{
|
||||
[MismatchCause.Inlining] = 15,
|
||||
[MismatchCause.SymbolVersioning] = 10,
|
||||
[MismatchCause.Unknown] = 29
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="NSubstitute" />
|
||||
<PackageReference Include="xunit.v3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.BinaryIndex.Validation\StellaOps.BinaryIndex.Validation.csproj" />
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.BinaryIndex.Validation.Abstractions\StellaOps.BinaryIndex.Validation.Abstractions.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,269 @@
|
||||
using FluentAssertions;
|
||||
using StellaOps.BinaryIndex.Validation.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.BinaryIndex.Validation.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for ValidationMetrics calculations.
|
||||
/// </summary>
|
||||
public class ValidationMetricsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(10, 0, 0.0)] // No positives
|
||||
[InlineData(10, 10, 1.0)] // All true positives
|
||||
[InlineData(5, 10, 0.667)] // Mixed
|
||||
public void Precision_CalculatedCorrectly(int fp, int tp, double expected)
|
||||
{
|
||||
// Arrange
|
||||
var metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 1,
|
||||
TotalFunctions = tp + fp,
|
||||
TruePositives = tp,
|
||||
FalsePositives = fp,
|
||||
TrueNegatives = 0,
|
||||
FalseNegatives = 0,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
};
|
||||
|
||||
// Assert
|
||||
metrics.Precision.Should().BeApproximately(expected, 0.01);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(10, 0, 0.0)] // No positives
|
||||
[InlineData(0, 10, 1.0)] // All true positives, no false negatives
|
||||
[InlineData(5, 10, 0.667)] // Mixed
|
||||
public void Recall_CalculatedCorrectly(int fn, int tp, double expected)
|
||||
{
|
||||
// Arrange
|
||||
var metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 1,
|
||||
TotalFunctions = tp + fn,
|
||||
TruePositives = tp,
|
||||
FalsePositives = 0,
|
||||
TrueNegatives = 0,
|
||||
FalseNegatives = fn,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
};
|
||||
|
||||
// Assert
|
||||
metrics.Recall.Should().BeApproximately(expected, 0.01);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void F1Score_HarmonicMean_OfPrecisionAndRecall()
|
||||
{
|
||||
// Arrange
|
||||
var metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 1,
|
||||
TotalFunctions = 100,
|
||||
TruePositives = 80,
|
||||
FalsePositives = 10,
|
||||
TrueNegatives = 5,
|
||||
FalseNegatives = 5,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
};
|
||||
|
||||
// Act
|
||||
var p = metrics.Precision;
|
||||
var r = metrics.Recall;
|
||||
var expectedF1 = 2 * (p * r) / (p + r);
|
||||
|
||||
// Assert
|
||||
metrics.F1Score.Should().BeApproximately(expectedF1, 0.001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Accuracy_CalculatedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 1,
|
||||
TotalFunctions = 100,
|
||||
TruePositives = 70,
|
||||
FalsePositives = 10,
|
||||
TrueNegatives = 15,
|
||||
FalseNegatives = 5,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
};
|
||||
|
||||
// Accuracy = (TP + TN) / Total = 85 / 100 = 0.85
|
||||
metrics.Accuracy.Should().Be(0.85);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MatchRate_CalculatedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 1,
|
||||
TotalFunctions = 100,
|
||||
TruePositives = 60,
|
||||
FalsePositives = 20,
|
||||
TrueNegatives = 10,
|
||||
FalseNegatives = 10,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
};
|
||||
|
||||
// MatchRate = (TP + FP) / Total = 80 / 100 = 0.80
|
||||
metrics.MatchRate.Should().Be(0.80);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FalsePositiveRate_CalculatedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 1,
|
||||
TotalFunctions = 100,
|
||||
TruePositives = 70,
|
||||
FalsePositives = 10,
|
||||
TrueNegatives = 15,
|
||||
FalseNegatives = 5,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
};
|
||||
|
||||
// FPR = FP / (FP + TN) = 10 / 25 = 0.4
|
||||
metrics.FalsePositiveRate.Should().Be(0.4);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void FalseNegativeRate_CalculatedCorrectly()
|
||||
{
|
||||
// Arrange
|
||||
var metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 1,
|
||||
TotalFunctions = 100,
|
||||
TruePositives = 70,
|
||||
FalsePositives = 10,
|
||||
TrueNegatives = 15,
|
||||
FalseNegatives = 5,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
};
|
||||
|
||||
// FNR = FN / (FN + TP) = 5 / 75 ≈ 0.0667
|
||||
metrics.FalseNegativeRate.Should().BeApproximately(0.0667, 0.001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AllZeros_ReturnsZeroMetrics()
|
||||
{
|
||||
// Arrange
|
||||
var metrics = new ValidationMetrics
|
||||
{
|
||||
TotalPairs = 0,
|
||||
TotalFunctions = 0,
|
||||
TruePositives = 0,
|
||||
FalsePositives = 0,
|
||||
TrueNegatives = 0,
|
||||
FalseNegatives = 0,
|
||||
MismatchCountsByBucket = new Dictionary<MismatchCause, int>()
|
||||
};
|
||||
|
||||
// Assert - should not throw, all should be 0
|
||||
metrics.Precision.Should().Be(0);
|
||||
metrics.Recall.Should().Be(0);
|
||||
metrics.F1Score.Should().Be(0);
|
||||
metrics.Accuracy.Should().Be(0);
|
||||
metrics.MatchRate.Should().Be(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests for ValidationConfig.
|
||||
/// </summary>
|
||||
public class ValidationConfigTests
|
||||
{
|
||||
[Fact]
|
||||
public void DefaultValues_AreReasonable()
|
||||
{
|
||||
// Arrange & Act
|
||||
var config = new ValidationConfig
|
||||
{
|
||||
Name = "Test Run",
|
||||
Matcher = new MatcherConfig { Type = MatcherType.SemanticDiff }
|
||||
};
|
||||
|
||||
// Assert
|
||||
config.MinMatchScore.Should().Be(0.5);
|
||||
config.MaxFalsePositiveRate.Should().Be(0.05);
|
||||
config.MaxFalseNegativeRate.Should().Be(0.10);
|
||||
config.MaxParallelism.Should().Be(4);
|
||||
config.IncludeMismatchAnalysis.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MatcherConfig_EnsembleWeights_DefaultEmpty()
|
||||
{
|
||||
// Arrange & Act
|
||||
var config = new MatcherConfig
|
||||
{
|
||||
Type = MatcherType.Ensemble
|
||||
};
|
||||
|
||||
// Assert
|
||||
config.EnsembleWeights.Should().BeEmpty();
|
||||
config.Options.Should().BeEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests for ValidationRun.
|
||||
/// </summary>
|
||||
public class ValidationRunTests
|
||||
{
|
||||
[Fact]
|
||||
public void Duration_CalculatedFromTimestamps()
|
||||
{
|
||||
// Arrange
|
||||
var start = DateTimeOffset.UtcNow;
|
||||
var end = start.AddMinutes(5).AddSeconds(30);
|
||||
|
||||
var run = new ValidationRun
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Config = new ValidationConfig
|
||||
{
|
||||
Name = "Test",
|
||||
Matcher = new MatcherConfig { Type = MatcherType.SemanticDiff }
|
||||
},
|
||||
Status = ValidationRunStatus.Completed,
|
||||
CreatedAt = start.AddMinutes(-1),
|
||||
StartedAt = start,
|
||||
CompletedAt = end
|
||||
};
|
||||
|
||||
// Assert
|
||||
run.Duration.Should().NotBeNull();
|
||||
run.Duration!.Value.TotalMinutes.Should().BeApproximately(5.5, 0.01);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Duration_NullWhenNotCompleted()
|
||||
{
|
||||
// Arrange
|
||||
var run = new ValidationRun
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Config = new ValidationConfig
|
||||
{
|
||||
Name = "Test",
|
||||
Matcher = new MatcherConfig { Type = MatcherType.SemanticDiff }
|
||||
},
|
||||
Status = ValidationRunStatus.Running,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
StartedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
// Assert
|
||||
run.Duration.Should().BeNull();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user