501 lines
18 KiB
C#
501 lines
18 KiB
C#
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// Copyright © 2025 StellaOps
|
|
// Sprint: SPRINT_8200_0012_0003_policy_engine_integration
|
|
// Task: PINT-8200-026 - Add snapshot tests for enriched verdict JSON structure
|
|
|
|
using System.Collections.Immutable;
|
|
using System.Text.Json;
|
|
using FluentAssertions;
|
|
using StellaOps.Policy.Engine.Attestation;
|
|
using StellaOps.Signals.EvidenceWeightedScore;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Policy.Engine.Tests.Snapshots;
|
|
|
|
/// <summary>
|
|
/// Snapshot tests for Evidence-Weighted Score (EWS) enriched verdict JSON structure.
|
|
/// Ensures EWS-enriched verdicts produce stable, auditor-facing JSON output.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// These tests validate:
|
|
/// - VerdictEvidenceWeightedScore JSON structure is stable
|
|
/// - Dimension breakdown order is deterministic (descending by contribution)
|
|
/// - Flags are sorted alphabetically
|
|
/// - ScoringProof contains all fields for reproducibility
|
|
/// - All components serialize correctly with proper JSON naming
|
|
/// </remarks>
|
|
public sealed class VerdictEwsSnapshotTests
|
|
{
|
|
private static readonly DateTimeOffset FrozenTime = DateTimeOffset.Parse("2025-12-24T12:00:00Z");
|
|
private static readonly JsonSerializerOptions JsonOptions = new()
|
|
{
|
|
WriteIndented = true,
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
|
};
|
|
|
|
#region VerdictEvidenceWeightedScore Snapshots
|
|
|
|
/// <summary>
|
|
/// Verifies that a high-score ActNow verdict produces stable canonical JSON.
|
|
/// </summary>
|
|
[Fact]
|
|
public void HighScoreActNow_ProducesStableCanonicalJson()
|
|
{
|
|
// Arrange
|
|
var ews = CreateHighScoreActNow();
|
|
|
|
// Act & Assert
|
|
var json = JsonSerializer.Serialize(ews, JsonOptions);
|
|
json.Should().NotBeNullOrWhiteSpace();
|
|
|
|
// Verify structure
|
|
ews.Score.Should().Be(92);
|
|
ews.Bucket.Should().Be("ActNow");
|
|
ews.Breakdown.Should().HaveCount(6);
|
|
ews.Flags.Should().Contain("kev");
|
|
ews.Flags.Should().Contain("live-signal");
|
|
ews.Proof.Should().NotBeNull();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that a medium-score ScheduleNext verdict produces stable canonical JSON.
|
|
/// </summary>
|
|
[Fact]
|
|
public void MediumScoreScheduleNext_ProducesStableCanonicalJson()
|
|
{
|
|
// Arrange
|
|
var ews = CreateMediumScoreScheduleNext();
|
|
|
|
// Act & Assert
|
|
var json = JsonSerializer.Serialize(ews, JsonOptions);
|
|
json.Should().NotBeNullOrWhiteSpace();
|
|
|
|
ews.Score.Should().Be(68);
|
|
ews.Bucket.Should().Be("ScheduleNext");
|
|
ews.Breakdown.Should().HaveCount(6);
|
|
ews.Flags.Should().BeEmpty();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that a low-score Watchlist verdict produces stable canonical JSON.
|
|
/// </summary>
|
|
[Fact]
|
|
public void LowScoreWatchlist_ProducesStableCanonicalJson()
|
|
{
|
|
// Arrange
|
|
var ews = CreateLowScoreWatchlist();
|
|
|
|
// Act & Assert
|
|
var json = JsonSerializer.Serialize(ews, JsonOptions);
|
|
json.Should().NotBeNullOrWhiteSpace();
|
|
|
|
ews.Score.Should().Be(18);
|
|
ews.Bucket.Should().Be("Watchlist");
|
|
ews.Flags.Should().Contain("vendor-na");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that VEX-mitigated verdict with low score produces stable JSON.
|
|
/// </summary>
|
|
[Fact]
|
|
public void VexMitigatedVerdict_ProducesStableCanonicalJson()
|
|
{
|
|
// Arrange
|
|
var ews = CreateVexMitigatedVerdict();
|
|
|
|
// Act & Assert
|
|
var json = JsonSerializer.Serialize(ews, JsonOptions);
|
|
json.Should().NotBeNullOrWhiteSpace();
|
|
|
|
ews.Score.Should().BeLessThan(30);
|
|
ews.Bucket.Should().Be("Watchlist");
|
|
ews.Flags.Should().Contain("vendor-na");
|
|
ews.Explanations.Should().Contain(e => e.Contains("VEX") || e.Contains("mitigated"));
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Breakdown Ordering Tests
|
|
|
|
/// <summary>
|
|
/// Verifies that breakdown dimensions are ordered by absolute contribution (descending).
|
|
/// </summary>
|
|
[Fact]
|
|
public void BreakdownOrder_IsSortedByContributionDescending()
|
|
{
|
|
// Arrange
|
|
var ews = CreateHighScoreActNow();
|
|
|
|
// Act
|
|
var contributions = ews.Breakdown.Select(b => Math.Abs(b.Contribution)).ToList();
|
|
|
|
// Assert - Each contribution should be >= the next
|
|
for (int i = 0; i < contributions.Count - 1; i++)
|
|
{
|
|
contributions[i].Should().BeGreaterOrEqualTo(contributions[i + 1],
|
|
$"Breakdown[{i}] contribution should be >= Breakdown[{i + 1}]");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that flags are sorted alphabetically.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Flags_AreSortedAlphabetically()
|
|
{
|
|
// Arrange
|
|
var ews = CreateHighScoreActNow();
|
|
|
|
// Act
|
|
var flags = ews.Flags.ToList();
|
|
|
|
// Assert
|
|
flags.Should().BeInAscendingOrder();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region ScoringProof Tests
|
|
|
|
/// <summary>
|
|
/// Verifies that ScoringProof contains all required fields for reproducibility.
|
|
/// </summary>
|
|
[Fact]
|
|
public void ScoringProof_ContainsAllRequiredFields()
|
|
{
|
|
// Arrange
|
|
var ews = CreateHighScoreActNow();
|
|
|
|
// Assert
|
|
ews.Proof.Should().NotBeNull();
|
|
ews.Proof!.Inputs.Should().NotBeNull();
|
|
ews.Proof.Weights.Should().NotBeNull();
|
|
ews.Proof.PolicyDigest.Should().NotBeNullOrWhiteSpace();
|
|
ews.Proof.CalculatorVersion.Should().NotBeNullOrWhiteSpace();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that ScoringProof inputs contain all 6 dimensions.
|
|
/// </summary>
|
|
[Fact]
|
|
public void ScoringProofInputs_ContainsAllDimensions()
|
|
{
|
|
// Arrange
|
|
var ews = CreateHighScoreActNow();
|
|
|
|
// Assert
|
|
var inputs = ews.Proof!.Inputs;
|
|
inputs.Reachability.Should().BeInRange(0.0, 1.0);
|
|
inputs.Runtime.Should().BeInRange(0.0, 1.0);
|
|
inputs.Backport.Should().BeInRange(0.0, 1.0);
|
|
inputs.Exploit.Should().BeInRange(0.0, 1.0);
|
|
inputs.SourceTrust.Should().BeInRange(0.0, 1.0);
|
|
inputs.Mitigation.Should().BeInRange(0.0, 1.0);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that ScoringProof weights sum to approximately 1.0.
|
|
/// </summary>
|
|
[Fact]
|
|
public void ScoringProofWeights_SumToOne()
|
|
{
|
|
// Arrange
|
|
var ews = CreateHighScoreActNow();
|
|
|
|
// Assert
|
|
var weights = ews.Proof!.Weights;
|
|
var sum = weights.Reachability + weights.Runtime + weights.Backport +
|
|
weights.Exploit + weights.SourceTrust + weights.Mitigation;
|
|
|
|
sum.Should().BeApproximately(1.0, 0.01, "Weights should sum to 1.0");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region JSON Serialization Tests
|
|
|
|
/// <summary>
|
|
/// Verifies that JSON uses camelCase property names.
|
|
/// </summary>
|
|
[Fact]
|
|
public void JsonSerialization_UsesCamelCasePropertyNames()
|
|
{
|
|
// Arrange
|
|
var ews = CreateHighScoreActNow();
|
|
|
|
// Act
|
|
var json = JsonSerializer.Serialize(ews, JsonOptions);
|
|
|
|
// Assert
|
|
json.Should().Contain("\"score\":");
|
|
json.Should().Contain("\"bucket\":");
|
|
json.Should().Contain("\"breakdown\":");
|
|
json.Should().Contain("\"flags\":");
|
|
json.Should().Contain("\"policyDigest\":");
|
|
json.Should().Contain("\"calculatedAt\":");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that null/empty fields are omitted from JSON.
|
|
/// </summary>
|
|
[Fact]
|
|
public void JsonSerialization_OmitsNullFields()
|
|
{
|
|
// Arrange
|
|
var ews = CreateMinimalVerdict();
|
|
|
|
// Act
|
|
var json = JsonSerializer.Serialize(ews, JsonOptions);
|
|
|
|
// Assert - These should be omitted when empty/null
|
|
if (ews.Guardrails is null)
|
|
{
|
|
json.Should().NotContain("\"guardrails\":");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies that timestamps are serialized in ISO-8601 format.
|
|
/// </summary>
|
|
[Fact]
|
|
public void JsonSerialization_TimestampsAreIso8601()
|
|
{
|
|
// Arrange
|
|
var ews = CreateHighScoreActNow();
|
|
|
|
// Act
|
|
var json = JsonSerializer.Serialize(ews, JsonOptions);
|
|
|
|
// Assert - ISO-8601 format with T separator
|
|
json.Should().MatchRegex(@"""calculatedAt"":\s*""\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies JSON serialization produces valid, parseable JSON structure.
|
|
/// Note: Full roundtrip deserialization is not supported due to JsonPropertyName
|
|
/// attributes differing from constructor parameter names in nested types.
|
|
/// Verdicts are created programmatically, not deserialized from external JSON.
|
|
/// </summary>
|
|
[Fact]
|
|
public void JsonSerialization_ProducesValidJsonStructure()
|
|
{
|
|
// Arrange
|
|
var original = CreateHighScoreActNow();
|
|
|
|
// Act
|
|
var json = JsonSerializer.Serialize(original, JsonOptions);
|
|
|
|
// Assert - JSON should be valid and contain expected structure
|
|
json.Should().NotBeNullOrWhiteSpace();
|
|
|
|
// Parse as JsonDocument to verify structure
|
|
using var doc = JsonDocument.Parse(json);
|
|
var root = doc.RootElement;
|
|
|
|
root.GetProperty("score").GetInt32().Should().Be(original.Score);
|
|
root.GetProperty("bucket").GetString().Should().Be(original.Bucket);
|
|
root.TryGetProperty("flags", out var flagsElement).Should().BeTrue();
|
|
root.TryGetProperty("policyDigest", out _).Should().BeTrue();
|
|
root.TryGetProperty("breakdown", out var breakdownElement).Should().BeTrue();
|
|
breakdownElement.GetArrayLength().Should().Be(original.Breakdown.Length);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Guardrails Tests
|
|
|
|
/// <summary>
|
|
/// Verifies that guardrails are correctly serialized when present.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Guardrails_WhenPresent_AreSerializedCorrectly()
|
|
{
|
|
// Arrange
|
|
var ews = CreateVerdictWithGuardrails();
|
|
|
|
// Act
|
|
var json = JsonSerializer.Serialize(ews, JsonOptions);
|
|
|
|
// Assert
|
|
ews.Guardrails.Should().NotBeNull();
|
|
json.Should().Contain("\"guardrails\":");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Factory Methods
|
|
|
|
private static VerdictEvidenceWeightedScore CreateHighScoreActNow()
|
|
{
|
|
return new VerdictEvidenceWeightedScore(
|
|
score: 92,
|
|
bucket: "ActNow",
|
|
breakdown:
|
|
[
|
|
new VerdictDimensionContribution("RuntimeSignal", "Rts", 28.0, 0.30, 0.93, false),
|
|
new VerdictDimensionContribution("Reachability", "Rch", 24.0, 0.25, 0.96, false),
|
|
new VerdictDimensionContribution("ExploitMaturity", "Xpl", 15.0, 0.15, 1.00, false),
|
|
new VerdictDimensionContribution("SourceTrust", "Src", 13.0, 0.15, 0.87, false),
|
|
new VerdictDimensionContribution("BackportStatus", "Bkp", 10.0, 0.10, 1.00, false),
|
|
new VerdictDimensionContribution("MitigationStatus", "Mit", 2.0, 0.05, 0.40, false)
|
|
],
|
|
flags: ["live-signal", "kev", "proven-path"],
|
|
explanations:
|
|
[
|
|
"KEV: Known Exploited Vulnerability (+15 floor)",
|
|
"Runtime signal detected in production environment",
|
|
"Call graph proves reachability to vulnerable function"
|
|
],
|
|
policyDigest: "sha256:abc123def456",
|
|
calculatedAt: FrozenTime,
|
|
guardrails: new VerdictAppliedGuardrails(
|
|
speculativeCap: false,
|
|
notAffectedCap: false,
|
|
runtimeFloor: true,
|
|
originalScore: 88,
|
|
adjustedScore: 92),
|
|
proof: CreateScoringProof(0.96, 0.93, 1.0, 1.0, 0.87, 0.40));
|
|
}
|
|
|
|
private static VerdictEvidenceWeightedScore CreateMediumScoreScheduleNext()
|
|
{
|
|
return new VerdictEvidenceWeightedScore(
|
|
score: 68,
|
|
bucket: "ScheduleNext",
|
|
breakdown:
|
|
[
|
|
new VerdictDimensionContribution("Reachability", "Rch", 20.0, 0.25, 0.80, false),
|
|
new VerdictDimensionContribution("RuntimeSignal", "Rts", 18.0, 0.30, 0.60, false),
|
|
new VerdictDimensionContribution("ExploitMaturity", "Xpl", 12.0, 0.15, 0.80, false),
|
|
new VerdictDimensionContribution("SourceTrust", "Src", 10.0, 0.15, 0.67, false),
|
|
new VerdictDimensionContribution("BackportStatus", "Bkp", 5.0, 0.10, 0.50, false),
|
|
new VerdictDimensionContribution("MitigationStatus", "Mit", 3.0, 0.05, 0.60, false)
|
|
],
|
|
flags: [],
|
|
explanations:
|
|
[
|
|
"Moderate reachability evidence from static analysis",
|
|
"No runtime signals detected"
|
|
],
|
|
policyDigest: "sha256:def789abc012",
|
|
calculatedAt: FrozenTime,
|
|
proof: CreateScoringProof(0.80, 0.60, 0.50, 0.80, 0.67, 0.60));
|
|
}
|
|
|
|
private static VerdictEvidenceWeightedScore CreateLowScoreWatchlist()
|
|
{
|
|
return new VerdictEvidenceWeightedScore(
|
|
score: 18,
|
|
bucket: "Watchlist",
|
|
breakdown:
|
|
[
|
|
new VerdictDimensionContribution("SourceTrust", "Src", 8.0, 0.15, 0.53, false),
|
|
new VerdictDimensionContribution("Reachability", "Rch", 5.0, 0.25, 0.20, false),
|
|
new VerdictDimensionContribution("ExploitMaturity", "Xpl", 3.0, 0.15, 0.20, false),
|
|
new VerdictDimensionContribution("RuntimeSignal", "Rts", 2.0, 0.30, 0.07, false),
|
|
new VerdictDimensionContribution("BackportStatus", "Bkp", 0.0, 0.10, 0.00, false),
|
|
new VerdictDimensionContribution("MitigationStatus", "Mit", 0.0, 0.05, 0.00, true)
|
|
],
|
|
flags: ["vendor-na"],
|
|
explanations:
|
|
[
|
|
"Vendor confirms not affected (VEX)",
|
|
"Low reachability - function not in call path"
|
|
],
|
|
policyDigest: "sha256:ghi345jkl678",
|
|
calculatedAt: FrozenTime,
|
|
proof: CreateScoringProof(0.20, 0.07, 0.0, 0.20, 0.53, 0.0));
|
|
}
|
|
|
|
private static VerdictEvidenceWeightedScore CreateVexMitigatedVerdict()
|
|
{
|
|
return new VerdictEvidenceWeightedScore(
|
|
score: 12,
|
|
bucket: "Watchlist",
|
|
breakdown:
|
|
[
|
|
new VerdictDimensionContribution("SourceTrust", "Src", 10.0, 0.15, 0.67, false),
|
|
new VerdictDimensionContribution("Reachability", "Rch", 2.0, 0.25, 0.08, false),
|
|
new VerdictDimensionContribution("ExploitMaturity", "Xpl", 0.0, 0.15, 0.00, false),
|
|
new VerdictDimensionContribution("RuntimeSignal", "Rts", 0.0, 0.30, 0.00, false),
|
|
new VerdictDimensionContribution("BackportStatus", "Bkp", 0.0, 0.10, 0.00, false),
|
|
new VerdictDimensionContribution("MitigationStatus", "Mit", 0.0, 0.05, 0.00, true)
|
|
],
|
|
flags: ["vendor-na"],
|
|
explanations:
|
|
[
|
|
"VEX: Vendor confirms not_affected status",
|
|
"Mitigation: Component not used in vulnerable context"
|
|
],
|
|
policyDigest: "sha256:mno901pqr234",
|
|
calculatedAt: FrozenTime,
|
|
guardrails: new VerdictAppliedGuardrails(
|
|
speculativeCap: false,
|
|
notAffectedCap: true,
|
|
runtimeFloor: false,
|
|
originalScore: 25,
|
|
adjustedScore: 12),
|
|
proof: CreateScoringProof(0.08, 0.0, 0.0, 0.0, 0.67, 0.0));
|
|
}
|
|
|
|
private static VerdictEvidenceWeightedScore CreateMinimalVerdict()
|
|
{
|
|
return new VerdictEvidenceWeightedScore(
|
|
score: 50,
|
|
bucket: "Investigate",
|
|
policyDigest: "sha256:minimal123");
|
|
}
|
|
|
|
private static VerdictEvidenceWeightedScore CreateVerdictWithGuardrails()
|
|
{
|
|
return new VerdictEvidenceWeightedScore(
|
|
score: 85,
|
|
bucket: "ActNow",
|
|
breakdown:
|
|
[
|
|
new VerdictDimensionContribution("RuntimeSignal", "Rts", 25.0, 0.30, 0.83, false),
|
|
new VerdictDimensionContribution("Reachability", "Rch", 20.0, 0.25, 0.80, false),
|
|
new VerdictDimensionContribution("ExploitMaturity", "Xpl", 15.0, 0.15, 1.00, false),
|
|
new VerdictDimensionContribution("SourceTrust", "Src", 12.0, 0.15, 0.80, false),
|
|
new VerdictDimensionContribution("BackportStatus", "Bkp", 8.0, 0.10, 0.80, false),
|
|
new VerdictDimensionContribution("MitigationStatus", "Mit", 5.0, 0.05, 1.00, false)
|
|
],
|
|
flags: ["kev"],
|
|
explanations: ["KEV: Known Exploited Vulnerability"],
|
|
policyDigest: "sha256:guardrails456",
|
|
calculatedAt: FrozenTime,
|
|
guardrails: new VerdictAppliedGuardrails(
|
|
speculativeCap: false,
|
|
notAffectedCap: false,
|
|
runtimeFloor: true,
|
|
originalScore: 80,
|
|
adjustedScore: 85),
|
|
proof: CreateScoringProof(0.80, 0.83, 0.80, 1.0, 0.80, 1.0));
|
|
}
|
|
|
|
private static VerdictScoringProof CreateScoringProof(
|
|
double rch, double rts, double bkp, double xpl, double src, double mit)
|
|
{
|
|
return new VerdictScoringProof(
|
|
inputs: new VerdictEvidenceInputs(
|
|
reachability: rch,
|
|
runtime: rts,
|
|
backport: bkp,
|
|
exploit: xpl,
|
|
sourceTrust: src,
|
|
mitigation: mit),
|
|
weights: new VerdictEvidenceWeights(
|
|
reachability: 0.25,
|
|
runtime: 0.30,
|
|
backport: 0.10,
|
|
exploit: 0.15,
|
|
sourceTrust: 0.15,
|
|
mitigation: 0.05),
|
|
policyDigest: "sha256:policy-v1",
|
|
calculatorVersion: "ews.v1.0.0",
|
|
calculatedAt: FrozenTime);
|
|
}
|
|
|
|
#endregion
|
|
}
|