UI work to fill SBOM sourcing management gap. UI planning remaining functionality exposure. Work on CI/Tests stabilization
Introduces CGS determinism test runs to CI workflows for Windows, macOS, Linux, Alpine, and Debian, fulfilling CGS-008 cross-platform requirements. Updates local-ci scripts to support new smoke steps, test timeouts, progress intervals, and project slicing for improved test isolation and diagnostics.
This commit is contained in:
master
@@ -0,0 +1,466 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// BackportVerdictDeterminismTests.cs
|
||||
// Sprint: SPRINT_20251229_004_002_BE_backport_status_service (BP-010)
|
||||
// Task: Add determinism tests for verdict stability
|
||||
// Description: Verify that same inputs produce same verdicts across multiple runs
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using StellaOps.Concelier.BackportProof.Models;
|
||||
using StellaOps.Concelier.BackportProof.Repositories;
|
||||
using StellaOps.Concelier.BackportProof.Services;
|
||||
using StellaOps.TestKit;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace StellaOps.Concelier.Core.Tests.BackportProof;
|
||||
|
||||
/// <summary>
|
||||
/// Determinism tests for Backport Status Service.
|
||||
/// Validates that:
|
||||
/// - Same input always produces identical verdict
|
||||
/// - Rule evaluation order doesn't matter
|
||||
/// - Confidence scoring is stable
|
||||
/// - JSON serialization is deterministic
|
||||
/// </summary>
|
||||
[Trait("Category", TestCategories.Determinism)]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
public sealed class BackportVerdictDeterminismTests
|
||||
{
|
||||
private static readonly DateTimeOffset FixedTimestamp = DateTimeOffset.Parse("2025-01-01T00:00:00Z");
|
||||
private readonly ITestOutputHelper _output;
|
||||
|
||||
public BackportVerdictDeterminismTests(ITestOutputHelper output)
|
||||
{
|
||||
_output = output;
|
||||
}
|
||||
|
||||
#region Same Input → Same Verdict Tests
|
||||
|
||||
[Fact]
|
||||
public async Task SameInput_ProducesIdenticalVerdict_Across10Iterations()
|
||||
{
|
||||
// Arrange
|
||||
var context = new ProductContext("debian", "bookworm", null, null);
|
||||
var package = new InstalledPackage(
|
||||
Key: new PackageKey(PackageEcosystem.Deb, "curl", "curl"),
|
||||
InstalledVersion: "7.88.1-10+deb12u5",
|
||||
BuildDigest: null,
|
||||
BuildId: null,
|
||||
SourcePackage: "curl");
|
||||
|
||||
var cve = "CVE-2024-1234";
|
||||
|
||||
var rules = CreateTestRules(context, package.Key, cve);
|
||||
var repository = CreateMockRepository(rules);
|
||||
var service = new BackportStatusService(repository, NullLogger<BackportStatusService>.Instance);
|
||||
|
||||
var verdicts = new List<string>();
|
||||
|
||||
// Act - Run 10 times
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
var verdict = await service.EvalPatchedStatusAsync(context, package, cve);
|
||||
var json = System.Text.Json.JsonSerializer.Serialize(verdict,
|
||||
new System.Text.Json.JsonSerializerOptions { WriteIndented = false });
|
||||
verdicts.Add(json);
|
||||
|
||||
_output.WriteLine($"Iteration {i + 1}: {json}");
|
||||
}
|
||||
|
||||
// Assert - All verdicts should be identical
|
||||
verdicts.Distinct().Should().HaveCount(1,
|
||||
"same input should produce identical verdict across all iterations");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DifferentRuleOrder_ProducesSameVerdict()
|
||||
{
|
||||
// Arrange
|
||||
var context = new ProductContext("alpine", "3.19", "main", null);
|
||||
var package = new InstalledPackage(
|
||||
Key: new PackageKey(PackageEcosystem.Apk, "openssl", "openssl"),
|
||||
InstalledVersion: "3.1.4-r5",
|
||||
BuildDigest: null,
|
||||
BuildId: null,
|
||||
SourcePackage: null);
|
||||
|
||||
var cve = "CVE-2024-5678";
|
||||
|
||||
// Create rules in different orders
|
||||
var rulesOrder1 = CreateTestRules(context, package.Key, cve).ToList();
|
||||
var rulesOrder2 = rulesOrder1.AsEnumerable().Reverse().ToList();
|
||||
var rulesOrder3 = rulesOrder1.OrderBy(_ => Guid.NewGuid()).ToList();
|
||||
|
||||
var repository1 = CreateMockRepository(rulesOrder1);
|
||||
var repository2 = CreateMockRepository(rulesOrder2);
|
||||
var repository3 = CreateMockRepository(rulesOrder3);
|
||||
|
||||
var service1 = new BackportStatusService(repository1, NullLogger<BackportStatusService>.Instance);
|
||||
var service2 = new BackportStatusService(repository2, NullLogger<BackportStatusService>.Instance);
|
||||
var service3 = new BackportStatusService(repository3, NullLogger<BackportStatusService>.Instance);
|
||||
|
||||
// Act
|
||||
var verdict1 = await service1.EvalPatchedStatusAsync(context, package, cve);
|
||||
var verdict2 = await service2.EvalPatchedStatusAsync(context, package, cve);
|
||||
var verdict3 = await service3.EvalPatchedStatusAsync(context, package, cve);
|
||||
|
||||
// Assert - All should produce same status and confidence
|
||||
verdict1.Status.Should().Be(verdict2.Status);
|
||||
verdict1.Status.Should().Be(verdict3.Status);
|
||||
verdict1.Confidence.Should().Be(verdict2.Confidence);
|
||||
verdict1.Confidence.Should().Be(verdict3.Confidence);
|
||||
verdict1.HasConflict.Should().Be(verdict2.HasConflict);
|
||||
verdict1.HasConflict.Should().Be(verdict3.HasConflict);
|
||||
|
||||
_output.WriteLine($"Status: {verdict1.Status}, Confidence: {verdict1.Confidence}, Conflict: {verdict1.HasConflict}");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Confidence Scoring Determinism
|
||||
|
||||
[Theory]
|
||||
[InlineData("7.88.1-10+deb12u5", FixStatus.Patched, VerdictConfidence.High)]
|
||||
[InlineData("7.88.1-10+deb12u4", FixStatus.Vulnerable, VerdictConfidence.High)]
|
||||
[InlineData("7.88.1-10+deb12u3", FixStatus.Vulnerable, VerdictConfidence.High)]
|
||||
public async Task BoundaryRule_ProducesConsistentConfidence(
|
||||
string installedVersion,
|
||||
FixStatus expectedStatus,
|
||||
VerdictConfidence expectedConfidence)
|
||||
{
|
||||
// Arrange
|
||||
var context = new ProductContext("debian", "bookworm", null, null);
|
||||
var package = new InstalledPackage(
|
||||
Key: new PackageKey(PackageEcosystem.Deb, "curl", "curl"),
|
||||
InstalledVersion: installedVersion,
|
||||
BuildDigest: null,
|
||||
BuildId: null,
|
||||
SourcePackage: "curl");
|
||||
|
||||
var cve = "CVE-2024-1234";
|
||||
|
||||
// Single boundary rule: fixed in 7.88.1-10+deb12u5
|
||||
var rules = new List<FixRule>
|
||||
{
|
||||
new BoundaryRule
|
||||
{
|
||||
RuleId = "debian-bookworm-curl-cve-2024-1234",
|
||||
Cve = cve,
|
||||
Context = context,
|
||||
Package = package.Key,
|
||||
Priority = RulePriority.DistroNative,
|
||||
Confidence = 0.95m,
|
||||
Evidence = new EvidencePointer(
|
||||
"debian-tracker",
|
||||
"https://security-tracker.debian.org/tracker/CVE-2024-1234",
|
||||
"sha256:abc123",
|
||||
FixedTimestamp),
|
||||
FixedVersion = "7.88.1-10+deb12u5"
|
||||
}
|
||||
};
|
||||
|
||||
var repository = CreateMockRepository(rules);
|
||||
var service = new BackportStatusService(repository, NullLogger<BackportStatusService>.Instance);
|
||||
|
||||
var verdicts = new List<BackportVerdict>();
|
||||
|
||||
// Act - Run 5 times
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
verdicts.Add(await service.EvalPatchedStatusAsync(context, package, cve));
|
||||
}
|
||||
|
||||
// Assert - All should have same status and confidence
|
||||
verdicts.Should().AllSatisfy(v =>
|
||||
{
|
||||
v.Status.Should().Be(expectedStatus);
|
||||
v.Confidence.Should().Be(expectedConfidence);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConflictingRules_AlwaysProducesMediumConfidence()
|
||||
{
|
||||
// Arrange
|
||||
var context = new ProductContext("debian", "bookworm", null, null);
|
||||
var package = new InstalledPackage(
|
||||
Key: new PackageKey(PackageEcosystem.Deb, "nginx", "nginx"),
|
||||
InstalledVersion: "1.24.0-1",
|
||||
BuildDigest: null,
|
||||
BuildId: null,
|
||||
SourcePackage: "nginx");
|
||||
|
||||
var cve = "CVE-2024-9999";
|
||||
|
||||
// Two conflicting rules from same priority
|
||||
var rules = new List<FixRule>
|
||||
{
|
||||
new BoundaryRule
|
||||
{
|
||||
RuleId = "rule-1",
|
||||
Cve = cve,
|
||||
Context = context,
|
||||
Package = package.Key,
|
||||
Priority = RulePriority.DistroNative,
|
||||
Confidence = 0.95m,
|
||||
Evidence = new EvidencePointer(
|
||||
"source-a",
|
||||
"https://example.com/a",
|
||||
null,
|
||||
FixedTimestamp),
|
||||
FixedVersion = "1.24.0-2" // Says fixed in -2
|
||||
},
|
||||
new BoundaryRule
|
||||
{
|
||||
RuleId = "rule-2",
|
||||
Cve = cve,
|
||||
Context = context,
|
||||
Package = package.Key,
|
||||
Priority = RulePriority.DistroNative,
|
||||
Confidence = 0.95m,
|
||||
Evidence = new EvidencePointer(
|
||||
"source-b",
|
||||
"https://example.com/b",
|
||||
null,
|
||||
FixedTimestamp),
|
||||
FixedVersion = "1.24.0-3" // Says fixed in -3 (conflict!)
|
||||
}
|
||||
};
|
||||
|
||||
var repository = CreateMockRepository(rules);
|
||||
var service = new BackportStatusService(repository, NullLogger<BackportStatusService>.Instance);
|
||||
|
||||
var verdicts = new List<BackportVerdict>();
|
||||
|
||||
// Act - Run 10 times
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
verdicts.Add(await service.EvalPatchedStatusAsync(context, package, cve));
|
||||
}
|
||||
|
||||
// Assert - All should have Medium confidence due to conflict
|
||||
verdicts.Should().AllSatisfy(v =>
|
||||
{
|
||||
v.Confidence.Should().Be(VerdictConfidence.Medium,
|
||||
"conflicting rules should always produce medium confidence");
|
||||
v.HasConflict.Should().BeTrue();
|
||||
v.ConflictReason.Should().NotBeNullOrEmpty();
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Edge Case Determinism
|
||||
|
||||
[Fact]
|
||||
public async Task NoRules_AlwaysReturnsUnknownLow()
|
||||
{
|
||||
// Arrange
|
||||
var context = new ProductContext("debian", "bookworm", null, null);
|
||||
var package = new InstalledPackage(
|
||||
Key: new PackageKey(PackageEcosystem.Deb, "unknown-package", null),
|
||||
InstalledVersion: "1.0.0",
|
||||
BuildDigest: null,
|
||||
BuildId: null,
|
||||
SourcePackage: null);
|
||||
|
||||
var cve = "CVE-2024-UNKNOWN";
|
||||
|
||||
var repository = CreateMockRepository(Array.Empty<FixRule>());
|
||||
var service = new BackportStatusService(repository, NullLogger<BackportStatusService>.Instance);
|
||||
|
||||
var verdicts = new List<BackportVerdict>();
|
||||
|
||||
// Act - Run 10 times with no rules
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
verdicts.Add(await service.EvalPatchedStatusAsync(context, package, cve));
|
||||
}
|
||||
|
||||
// Assert
|
||||
verdicts.Should().AllSatisfy(v =>
|
||||
{
|
||||
v.Status.Should().Be(FixStatus.Unknown);
|
||||
v.Confidence.Should().Be(VerdictConfidence.Low);
|
||||
v.HasConflict.Should().BeFalse();
|
||||
v.AppliedRuleIds.Should().BeEmpty();
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NotAffected_AlwaysWinsImmediately()
|
||||
{
|
||||
// Arrange
|
||||
var context = new ProductContext("debian", "bookworm", null, null);
|
||||
var package = new InstalledPackage(
|
||||
Key: new PackageKey(PackageEcosystem.Deb, "systemd", "systemd"),
|
||||
InstalledVersion: "252.19-1~deb12u1",
|
||||
BuildDigest: null,
|
||||
BuildId: null,
|
||||
SourcePackage: "systemd");
|
||||
|
||||
var cve = "CVE-2024-SERVER-ONLY";
|
||||
|
||||
// Not-affected rule + other rules (not-affected should win)
|
||||
var rules = new List<FixRule>
|
||||
{
|
||||
new StatusRule
|
||||
{
|
||||
RuleId = "not-affected-rule",
|
||||
Cve = cve,
|
||||
Context = context,
|
||||
Package = package.Key,
|
||||
Priority = RulePriority.DistroNative,
|
||||
Confidence = 1.0m,
|
||||
Evidence = new EvidencePointer(
|
||||
"debian-tracker",
|
||||
"https://security-tracker.debian.org/tracker/CVE-2024-SERVER-ONLY",
|
||||
null,
|
||||
FixedTimestamp),
|
||||
Status = FixStatus.NotAffected
|
||||
},
|
||||
new BoundaryRule
|
||||
{
|
||||
RuleId = "boundary-rule",
|
||||
Cve = cve,
|
||||
Context = context,
|
||||
Package = package.Key,
|
||||
Priority = RulePriority.ThirdParty,
|
||||
Confidence = 0.7m,
|
||||
Evidence = new EvidencePointer(
|
||||
"nvd",
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2024-SERVER-ONLY",
|
||||
null,
|
||||
FixedTimestamp),
|
||||
FixedVersion = "252.20-1"
|
||||
}
|
||||
};
|
||||
|
||||
var repository = CreateMockRepository(rules);
|
||||
var service = new BackportStatusService(repository, NullLogger<BackportStatusService>.Instance);
|
||||
|
||||
var verdicts = new List<BackportVerdict>();
|
||||
|
||||
// Act - Run 10 times
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
verdicts.Add(await service.EvalPatchedStatusAsync(context, package, cve));
|
||||
}
|
||||
|
||||
// Assert - NotAffected should always win with High confidence
|
||||
verdicts.Should().AllSatisfy(v =>
|
||||
{
|
||||
v.Status.Should().Be(FixStatus.NotAffected);
|
||||
v.Confidence.Should().Be(VerdictConfidence.High);
|
||||
v.AppliedRuleIds.Should().Contain("not-affected-rule");
|
||||
});
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region JSON Serialization Determinism
|
||||
|
||||
[Fact]
|
||||
public async Task JsonSerialization_IsStable()
|
||||
{
|
||||
// Arrange
|
||||
var context = new ProductContext("alpine", "3.19", "main", "x86_64");
|
||||
var package = new InstalledPackage(
|
||||
Key: new PackageKey(PackageEcosystem.Apk, "busybox", "busybox"),
|
||||
InstalledVersion: "1.36.1-r15",
|
||||
BuildDigest: "sha256:abcdef1234567890",
|
||||
BuildId: "build-123",
|
||||
SourcePackage: null);
|
||||
|
||||
var cve = "CVE-2024-JSON-TEST";
|
||||
|
||||
var rules = CreateTestRules(context, package.Key, cve);
|
||||
var repository = CreateMockRepository(rules);
|
||||
var service = new BackportStatusService(repository, NullLogger<BackportStatusService>.Instance);
|
||||
|
||||
var jsonOutputs = new List<string>();
|
||||
|
||||
// Act - Serialize 10 times
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
var verdict = await service.EvalPatchedStatusAsync(context, package, cve);
|
||||
var json = System.Text.Json.JsonSerializer.Serialize(verdict,
|
||||
new System.Text.Json.JsonSerializerOptions
|
||||
{
|
||||
WriteIndented = false,
|
||||
PropertyNamingPolicy = System.Text.Json.JsonNamingPolicy.CamelCase
|
||||
});
|
||||
jsonOutputs.Add(json);
|
||||
}
|
||||
|
||||
// Assert - All JSON should be byte-identical
|
||||
jsonOutputs.Distinct().Should().HaveCount(1,
|
||||
"JSON serialization should be deterministic");
|
||||
|
||||
_output.WriteLine($"Deterministic JSON: {jsonOutputs[0]}");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static List<FixRule> CreateTestRules(
|
||||
ProductContext context,
|
||||
PackageKey package,
|
||||
string cve)
|
||||
{
|
||||
return new List<FixRule>
|
||||
{
|
||||
new BoundaryRule
|
||||
{
|
||||
RuleId = $"rule-{cve}-1",
|
||||
Cve = cve,
|
||||
Context = context,
|
||||
Package = package,
|
||||
Priority = RulePriority.DistroNative,
|
||||
Confidence = 0.95m,
|
||||
Evidence = new EvidencePointer(
|
||||
"test-source",
|
||||
"https://example.com/advisory",
|
||||
"sha256:test123",
|
||||
DateTimeOffset.Parse("2025-01-01T00:00:00Z")),
|
||||
FixedVersion = "1.36.1-r16"
|
||||
},
|
||||
new BoundaryRule
|
||||
{
|
||||
RuleId = $"rule-{cve}-2",
|
||||
Cve = cve,
|
||||
Context = context,
|
||||
Package = package,
|
||||
Priority = RulePriority.VendorCsaf,
|
||||
Confidence = 0.90m,
|
||||
Evidence = new EvidencePointer(
|
||||
"vendor-csaf",
|
||||
"https://vendor.example.com/csaf",
|
||||
"sha256:vendor456",
|
||||
DateTimeOffset.Parse("2025-01-02T00:00:00Z")),
|
||||
FixedVersion = "1.36.1-r16"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static IFixRuleRepository CreateMockRepository(IEnumerable<FixRule> rules)
|
||||
{
|
||||
var mock = new Mock<IFixRuleRepository>();
|
||||
var rulesList = rules.ToList();
|
||||
|
||||
mock.Setup(r => r.GetRulesAsync(
|
||||
It.IsAny<ProductContext>(),
|
||||
It.IsAny<PackageKey>(),
|
||||
It.IsAny<string>(),
|
||||
It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(rulesList);
|
||||
|
||||
return mock.Object;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -13,6 +13,7 @@
|
||||
<PackageReference Include="Moq" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.BackportProof/StellaOps.Concelier.BackportProof.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.RawModels/StellaOps.Concelier.RawModels.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
@@ -20,4 +21,4 @@
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
<ProjectReference Include="../../../Aoc/__Libraries/StellaOps.Aoc/StellaOps.Aoc.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user