stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

synergy moats product advisory implementations

Browse Source
This commit is contained in:
master
2026-01-17 01:30:03 +02:00
parent 77ff029205
commit d8d9c0a6e3
106 changed files with 20603 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

490
src/Cli/__Tests/StellaOps.Cli.Tests/GoldenOutput/DeterminismReplayGoldenTests.cs
View File

@@ -489,6 +489,236 @@ public sealed class DeterminismReplayGoldenTests
#endregion
#region Explain Block Golden Tests (Sprint 026 - WHY-004)
/// <summary>
/// Verifies that explain block JSON output matches golden snapshot.
/// Sprint: SPRINT_20260117_026_CLI_why_blocked_command
/// </summary>
[Fact]
public void ExplainBlock_Json_MatchesGolden()
{
// Arrange
var explanation = CreateFrozenBlockExplanation();
// Act
var actual = JsonSerializer.Serialize(explanation, JsonOptions).NormalizeLf();
// Assert - Golden snapshot
var expected = """
{
"artifact": "sha256:abc123def456789012345678901234567890123456789012345678901234",
"status": "BLOCKED",
"gate": "VexTrust",
"reason": "Trust score below threshold (0.45 \u003C 0.70)",
"suggestion": "Obtain VEX statement from trusted issuer or add issuer to trust registry",
"evaluationTime": "2026-01-15T10:30:00+00:00",
"policyVersion": "v2.3.0",
"evidence": [
{
"type": "REACH",
"id": "reach:sha256:789abc123def456",
"source": "static-analysis",
"timestamp": "2026-01-15T08:00:00+00:00"
},
{
"type": "VEX",
"id": "vex:sha256:def456789abc123",
"source": "vendor-x",
"timestamp": "2026-01-15T09:00:00+00:00"
}
],
"replayCommand": "stella verify verdict --verdict urn:stella:verdict:sha256:abc123:v2.3.0:1737108000",
"replayToken": "urn:stella:verdict:sha256:abc123:v2.3.0:1737108000",
"evaluationTrace": [
{
"step": 1,
"gate": "SbomPresent",
"result": "PASS",
"durationMs": 15
},
{
"step": 2,
"gate": "VexTrust",
"result": "FAIL",
"durationMs": 45
},
{
"step": 3,
"gate": "VulnScan",
"result": "PASS",
"durationMs": 250
}
],
"determinismHash": "sha256:e3b0c44298fc1c14"
}
""".NormalizeLf();
actual.Should().Be(expected);
}
/// <summary>
/// Verifies that explain block table output matches golden snapshot.
/// </summary>
[Fact]
public void ExplainBlock_Table_MatchesGolden()
{
// Arrange
var explanation = CreateFrozenBlockExplanation();
// Act
var actual = FormatBlockExplanationTable(explanation, showEvidence: false, showTrace: false).NormalizeLf();
// Assert - Golden snapshot
var expected = """
Artifact: sha256:abc123def456789012345678901234567890123456789012345678901234
Status: BLOCKED
Gate: VexTrust
Reason: Trust score below threshold (0.45 < 0.70)
Suggestion: Obtain VEX statement from trusted issuer or add issuer to trust registry
Evidence:
[REACH ] reach:sha256...def456 static-analysis 2026-01-15T08:00:00Z
[VEX ] vex:sha256:d...bc123 vendor-x 2026-01-15T09:00:00Z
Replay: stella verify verdict --verdict urn:stella:verdict:sha256:abc123:v2.3.0:1737108000
""".NormalizeLf();
actual.Trim().Should().Be(expected.Trim());
}
/// <summary>
/// Verifies that explain block markdown output matches golden snapshot.
/// </summary>
[Fact]
public void ExplainBlock_Markdown_MatchesGolden()
{
// Arrange
var explanation = CreateFrozenBlockExplanation();
// Act
var actual = FormatBlockExplanationMarkdown(explanation, showEvidence: false, showTrace: false).NormalizeLf();
// Assert - Key elements present
actual.Should().Contain("## Block Explanation");
actual.Should().Contain("**Artifact:** `sha256:abc123def456789012345678901234567890123456789012345678901234`");
actual.Should().Contain("**Status:** BLOCKED");
actual.Should().Contain("### Gate Decision");
actual.Should().Contain("| Property | Value |");
actual.Should().Contain("| Gate | VexTrust |");
actual.Should().Contain("| Reason | Trust score below threshold");
actual.Should().Contain("### Evidence");
actual.Should().Contain("| Type | ID | Source | Timestamp |");
actual.Should().Contain("### Verification");
actual.Should().Contain("```bash");
actual.Should().Contain("stella verify verdict --verdict");
}
/// <summary>
/// Verifies that explain block with --show-trace includes evaluation trace.
/// </summary>
[Fact]
public void ExplainBlock_WithTrace_MatchesGolden()
{
// Arrange
var explanation = CreateFrozenBlockExplanation();
// Act
var actual = FormatBlockExplanationTable(explanation, showEvidence: false, showTrace: true).NormalizeLf();
// Assert
actual.Should().Contain("Evaluation Trace:");
actual.Should().Contain("1. SbomPresent");
actual.Should().Contain("PASS");
actual.Should().Contain("2. VexTrust");
actual.Should().Contain("FAIL");
actual.Should().Contain("3. VulnScan");
actual.Should().Contain("PASS");
}
/// <summary>
/// Verifies that same inputs produce identical outputs (byte-for-byte).
/// M2 moat requirement: Deterministic trace + referenced evidence artifacts.
/// </summary>
[Fact]
public void ExplainBlock_SameInputs_ProducesIdenticalOutput()
{
// Arrange
var exp1 = CreateFrozenBlockExplanation();
var exp2 = CreateFrozenBlockExplanation();
// Act
var json1 = JsonSerializer.Serialize(exp1, JsonOptions);
var json2 = JsonSerializer.Serialize(exp2, JsonOptions);
var table1 = FormatBlockExplanationTable(exp1, true, true);
var table2 = FormatBlockExplanationTable(exp2, true, true);
var md1 = FormatBlockExplanationMarkdown(exp1, true, true);
var md2 = FormatBlockExplanationMarkdown(exp2, true, true);
// Assert - All formats must be identical
json1.Should().Be(json2, "JSON output must be deterministic");
table1.Should().Be(table2, "Table output must be deterministic");
md1.Should().Be(md2, "Markdown output must be deterministic");
}
/// <summary>
/// Verifies that evidence is sorted by timestamp for deterministic ordering.
/// </summary>
[Fact]
public void ExplainBlock_EvidenceIsSortedByTimestamp()
{
// Arrange
var explanation = CreateFrozenBlockExplanation();
// Assert - Evidence should be sorted by timestamp (ascending)
var timestamps = explanation.Evidence.Select(e => e.Timestamp).ToList();
timestamps.Should().BeInAscendingOrder();
}
/// <summary>
/// Verifies that evaluation trace is sorted by step number.
/// </summary>
[Fact]
public void ExplainBlock_TraceIsSortedByStep()
{
// Arrange
var explanation = CreateFrozenBlockExplanation();
// Assert - Trace should be sorted by step number
var steps = explanation.EvaluationTrace.Select(t => t.Step).ToList();
steps.Should().BeInAscendingOrder();
}
/// <summary>
/// Verifies that not-blocked artifacts produce deterministic output.
/// </summary>
[Fact]
public void ExplainBlock_NotBlocked_MatchesGolden()
{
// Arrange
var explanation = CreateFrozenNotBlockedExplanation();
// Act
var actual = JsonSerializer.Serialize(explanation, JsonOptions).NormalizeLf();
// Assert - Golden snapshot for not blocked
var expected = """
{
"artifact": "sha256:fedcba9876543210",
"status": "NOT_BLOCKED",
"message": "Artifact passed all policy gates",
"gatesEvaluated": 5,
"evaluationTime": "2026-01-15T10:30:00+00:00",
"policyVersion": "v2.3.0"
}
""".NormalizeLf();
actual.Should().Be(expected);
}
#endregion
#region Cross-Platform Golden Tests
/// <summary>
@@ -753,6 +983,174 @@ public sealed class DeterminismReplayGoldenTests
explanation.DeterminismHash = $"sha256:{Convert.ToHexStringLower(hashBytes)[..16]}";
}
// Explain Block helpers (Sprint 026 - WHY-004)
private static BlockExplanation CreateFrozenBlockExplanation()
{
return new BlockExplanation
{
Artifact = "sha256:abc123def456789012345678901234567890123456789012345678901234",
Status = "BLOCKED",
Gate = "VexTrust",
Reason = "Trust score below threshold (0.45 < 0.70)",
Suggestion = "Obtain VEX statement from trusted issuer or add issuer to trust registry",
EvaluationTime = FixedTimestamp,
PolicyVersion = "v2.3.0",
Evidence =
[
new BlockEvidence
{
Type = "REACH",
Id = "reach:sha256:789abc123def456",
Source = "static-analysis",
Timestamp = FixedTimestamp.AddHours(-2.5) // 08:00
},
new BlockEvidence
{
Type = "VEX",
Id = "vex:sha256:def456789abc123",
Source = "vendor-x",
Timestamp = FixedTimestamp.AddHours(-1.5) // 09:00
}
],
ReplayCommand = "stella verify verdict --verdict urn:stella:verdict:sha256:abc123:v2.3.0:1737108000",
ReplayToken = "urn:stella:verdict:sha256:abc123:v2.3.0:1737108000",
EvaluationTrace =
[
new BlockTraceStep { Step = 1, Gate = "SbomPresent", Result = "PASS", DurationMs = 15 },
new BlockTraceStep { Step = 2, Gate = "VexTrust", Result = "FAIL", DurationMs = 45 },
new BlockTraceStep { Step = 3, Gate = "VulnScan", Result = "PASS", DurationMs = 250 }
],
DeterminismHash = "sha256:e3b0c44298fc1c14"
};
}
private static NotBlockedExplanation CreateFrozenNotBlockedExplanation()
{
return new NotBlockedExplanation
{
Artifact = "sha256:fedcba9876543210",
Status = "NOT_BLOCKED",
Message = "Artifact passed all policy gates",
GatesEvaluated = 5,
EvaluationTime = FixedTimestamp,
PolicyVersion = "v2.3.0"
};
}
private static string FormatBlockExplanationTable(BlockExplanation exp, bool showEvidence, bool showTrace)
{
var sb = new StringBuilder();
sb.AppendLine($"Artifact: {exp.Artifact}");
sb.AppendLine($"Status: {exp.Status}");
sb.AppendLine();
sb.AppendLine($"Gate: {exp.Gate}");
sb.AppendLine($"Reason: {exp.Reason}");
sb.AppendLine($"Suggestion: {exp.Suggestion}");
sb.AppendLine();
sb.AppendLine("Evidence:");
foreach (var evidence in exp.Evidence.OrderBy(e => e.Timestamp))
{
var truncatedId = TruncateBlockId(evidence.Id);
sb.AppendLine($" [{evidence.Type,-6}] {truncatedId,-20} {evidence.Source,-15} {evidence.Timestamp:yyyy-MM-ddTHH:mm:ssZ}");
}
if (showTrace && exp.EvaluationTrace.Count > 0)
{
sb.AppendLine();
sb.AppendLine("Evaluation Trace:");
foreach (var step in exp.EvaluationTrace.OrderBy(t => t.Step))
{
sb.AppendLine($" {step.Step}. {step.Gate,-15} {step.Result,-6} ({step.DurationMs}ms)");
}
}
if (showEvidence)
{
sb.AppendLine();
sb.AppendLine("Evidence Details:");
foreach (var evidence in exp.Evidence.OrderBy(e => e.Timestamp))
{
sb.AppendLine($" - Type: {evidence.Type}");
sb.AppendLine($" ID: {evidence.Id}");
sb.AppendLine($" Source: {evidence.Source}");
sb.AppendLine($" Retrieve: stella evidence get {evidence.Id}");
sb.AppendLine();
}
}
sb.AppendLine();
sb.AppendLine($"Replay: {exp.ReplayCommand}");
return sb.ToString();
}
private static string FormatBlockExplanationMarkdown(BlockExplanation exp, bool showEvidence, bool showTrace)
{
var sb = new StringBuilder();
sb.AppendLine("## Block Explanation");
sb.AppendLine();
sb.AppendLine($"**Artifact:** `{exp.Artifact}`");
sb.AppendLine($"**Status:** {exp.Status}");
sb.AppendLine();
sb.AppendLine("### Gate Decision");
sb.AppendLine();
sb.AppendLine("| Property | Value |");
sb.AppendLine("|----------|-------|");
sb.AppendLine($"| Gate | {exp.Gate} |");
sb.AppendLine($"| Reason | {exp.Reason} |");
sb.AppendLine($"| Suggestion | {exp.Suggestion} |");
sb.AppendLine($"| Policy Version | {exp.PolicyVersion} |");
sb.AppendLine();
sb.AppendLine("### Evidence");
sb.AppendLine();
sb.AppendLine("| Type | ID | Source | Timestamp |");
sb.AppendLine("|------|-----|--------|-----------|");
foreach (var evidence in exp.Evidence.OrderBy(e => e.Timestamp))
{
var truncatedId = TruncateBlockId(evidence.Id);
sb.AppendLine($"| {evidence.Type} | `{truncatedId}` | {evidence.Source} | {evidence.Timestamp:yyyy-MM-dd HH:mm} |");
}
sb.AppendLine();
if (showTrace && exp.EvaluationTrace.Count > 0)
{
sb.AppendLine("### Evaluation Trace");
sb.AppendLine();
sb.AppendLine("| Step | Gate | Result | Duration |");
sb.AppendLine("|------|------|--------|----------|");
foreach (var step in exp.EvaluationTrace.OrderBy(t => t.Step))
{
sb.AppendLine($"| {step.Step} | {step.Gate} | {step.Result} | {step.DurationMs}ms |");
}
sb.AppendLine();
}
sb.AppendLine("### Verification");
sb.AppendLine();
sb.AppendLine("```bash");
sb.AppendLine(exp.ReplayCommand);
sb.AppendLine("```");
return sb.ToString();
}
private static string TruncateBlockId(string id)
{
if (id.Length <= 20)
{
return id;
}
var prefix = id[..12];
var suffix = id[^6..];
return $"{prefix}...{suffix}";
}
#endregion
#region Test Models
@@ -934,6 +1332,98 @@ public sealed class DeterminismReplayGoldenTests
public string? Details { get; set; }
}
// Explain Block models (Sprint 026 - WHY-004)
private sealed class BlockExplanation
{
[JsonPropertyName("artifact")]
public string Artifact { get; set; } = string.Empty;
[JsonPropertyName("status")]
public string Status { get; set; } = string.Empty;
[JsonPropertyName("gate")]
public string Gate { get; set; } = string.Empty;
[JsonPropertyName("reason")]
public string Reason { get; set; } = string.Empty;
[JsonPropertyName("suggestion")]
public string Suggestion { get; set; } = string.Empty;
[JsonPropertyName("evaluationTime")]
public DateTimeOffset EvaluationTime { get; set; }
[JsonPropertyName("policyVersion")]
public string PolicyVersion { get; set; } = string.Empty;
[JsonPropertyName("evidence")]
public List<BlockEvidence> Evidence { get; set; } = [];
[JsonPropertyName("replayCommand")]
public string ReplayCommand { get; set; } = string.Empty;
[JsonPropertyName("replayToken")]
public string ReplayToken { get; set; } = string.Empty;
[JsonPropertyName("evaluationTrace")]
public List<BlockTraceStep> EvaluationTrace { get; set; } = [];
[JsonPropertyName("determinismHash")]
public string DeterminismHash { get; set; } = string.Empty;
}
private sealed class BlockEvidence
{
[JsonPropertyName("type")]
public string Type { get; set; } = string.Empty;
[JsonPropertyName("id")]
public string Id { get; set; } = string.Empty;
[JsonPropertyName("source")]
public string Source { get; set; } = string.Empty;
[JsonPropertyName("timestamp")]
public DateTimeOffset Timestamp { get; set; }
}
private sealed class BlockTraceStep
{
[JsonPropertyName("step")]
public int Step { get; set; }
[JsonPropertyName("gate")]
public string Gate { get; set; } = string.Empty;
[JsonPropertyName("result")]
public string Result { get; set; } = string.Empty;
[JsonPropertyName("durationMs")]
public int DurationMs { get; set; }
}
private sealed class NotBlockedExplanation
{
[JsonPropertyName("artifact")]
public string Artifact { get; set; } = string.Empty;
[JsonPropertyName("status")]
public string Status { get; set; } = string.Empty;
[JsonPropertyName("message")]
public string Message { get; set; } = string.Empty;
[JsonPropertyName("gatesEvaluated")]
public int GatesEvaluated { get; set; }
[JsonPropertyName("evaluationTime")]
public DateTimeOffset EvaluationTime { get; set; }
[JsonPropertyName("policyVersion")]
public string PolicyVersion { get; set; } = string.Empty;
}
#endregion
}