Complete batch 012 (golden set diff) and 013 (advisory chat), fix build errors
Sprints completed: - SPRINT_20260110_012_* (golden set diff layer - 10 sprints) - SPRINT_20260110_013_* (advisory chat - 4 sprints) Build fixes applied: - Fix namespace conflicts with Microsoft.Extensions.Options.Options.Create - Fix VexDecisionReachabilityIntegrationTests API drift (major rewrite) - Fix VexSchemaValidationTests FluentAssertions method name - Fix FixChainGateIntegrationTests ambiguous type references - Fix AdvisoryAI test files required properties and namespace aliases - Add stub types for CveMappingController (ICveSymbolMappingService) - Fix VerdictBuilderService static context issue Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
master
256
src/__Tests/Integration/GoldenSetDiff/CorpusValidationTests.cs
Normal file
256
src/__Tests/Integration/GoldenSetDiff/CorpusValidationTests.cs
Normal file
@@ -0,0 +1,256 @@
|
||||
// Licensed under AGPL-3.0-or-later. Copyright (C) 2026 StellaOps Contributors.
|
||||
// Sprint: SPRINT_20260110_012_010_TEST
|
||||
// Task: GTV-006 - Corpus Validation Test Suite
|
||||
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StellaOps.BinaryIndex.GoldenSet;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Integration.GoldenSetDiff;
|
||||
|
||||
/// <summary>
|
||||
/// Validates the golden set corpus for correctness, uniqueness, and completeness.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class CorpusValidationTests
|
||||
{
|
||||
private readonly string _corpusPath;
|
||||
private readonly IGoldenSetValidator _validator;
|
||||
|
||||
public CorpusValidationTests()
|
||||
{
|
||||
// Resolve corpus path relative to test assembly
|
||||
var assemblyLocation = Path.GetDirectoryName(typeof(CorpusValidationTests).Assembly.Location)!;
|
||||
_corpusPath = Path.Combine(assemblyLocation, "golden-sets");
|
||||
|
||||
// Create validator with mocked dependencies
|
||||
var sinkRegistry = new Mock<ISinkRegistry>();
|
||||
sinkRegistry.Setup(r => r.IsKnownSink(It.IsAny<string>())).Returns(true);
|
||||
|
||||
var options = Options.Create(new GoldenSetOptions
|
||||
{
|
||||
Validation = new GoldenSetValidationOptions
|
||||
{
|
||||
OfflineMode = true,
|
||||
ValidateSinks = false,
|
||||
StrictEdgeFormat = true
|
||||
}
|
||||
});
|
||||
|
||||
_validator = new GoldenSetValidator(
|
||||
sinkRegistry.Object,
|
||||
options,
|
||||
NullLogger<GoldenSetValidator>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AllGoldenSetsInCorpus_PassValidation()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
goldenSetFiles.Should().NotBeEmpty("corpus should contain golden set files");
|
||||
|
||||
// Act & Assert
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var result = await _validator.ValidateYamlAsync(yaml, new ValidationOptions
|
||||
{
|
||||
OfflineMode = true, // Don't hit CVE APIs during tests
|
||||
ValidateSinks = false, // Using mock registry
|
||||
StrictEdgeFormat = true
|
||||
});
|
||||
|
||||
result.IsValid.Should().BeTrue(
|
||||
$"Validation failed for {Path.GetFileName(file)}: {string.Join(", ", result.Errors.Select(e => e.Message))}");
|
||||
result.ContentDigest.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AllGoldenSets_HaveUniqueContentDigests()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
var digests = new Dictionary<string, string>();
|
||||
|
||||
// Act
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var result = await _validator.ValidateYamlAsync(yaml);
|
||||
|
||||
if (!result.IsValid)
|
||||
{
|
||||
continue; // Skip invalid files (tested separately)
|
||||
}
|
||||
|
||||
var digest = result.ContentDigest!;
|
||||
|
||||
// Assert
|
||||
if (digests.TryGetValue(digest, out var existingFile))
|
||||
{
|
||||
Assert.Fail($"Duplicate digest found: {Path.GetFileName(file)} and {Path.GetFileName(existingFile)}");
|
||||
}
|
||||
|
||||
digests[digest] = file;
|
||||
}
|
||||
|
||||
digests.Should().NotBeEmpty("should have computed digests for valid golden sets");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AllGoldenSets_HaveRequiredMetadata()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
// Act & Assert
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
|
||||
// Parse using the serializer
|
||||
var definition = GoldenSetYamlSerializer.Deserialize(yaml);
|
||||
|
||||
// Verify required metadata
|
||||
definition.Metadata.AuthorId.Should().NotBeNullOrWhiteSpace(
|
||||
$"Golden set {Path.GetFileName(file)} should have author_id");
|
||||
definition.Metadata.CreatedAt.Should().NotBe(default,
|
||||
$"Golden set {Path.GetFileName(file)} should have created_at");
|
||||
definition.Metadata.SourceRef.Should().NotBeNullOrWhiteSpace(
|
||||
$"Golden set {Path.GetFileName(file)} should have source_ref");
|
||||
definition.Metadata.Tags.Should().NotBeEmpty(
|
||||
$"Golden set {Path.GetFileName(file)} should have at least one tag");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AllGoldenSets_HaveValidTargets()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
// Act & Assert
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var definition = GoldenSetYamlSerializer.Deserialize(yaml);
|
||||
|
||||
definition.Targets.Should().NotBeEmpty(
|
||||
$"Golden set {Path.GetFileName(file)} should have at least one target");
|
||||
|
||||
foreach (var target in definition.Targets)
|
||||
{
|
||||
target.FunctionName.Should().NotBeNullOrWhiteSpace(
|
||||
$"Target in {Path.GetFileName(file)} should have function name");
|
||||
|
||||
// Either edges or sinks should be present
|
||||
var hasEdges = !target.Edges.IsDefaultOrEmpty;
|
||||
var hasSinks = !target.Sinks.IsDefaultOrEmpty;
|
||||
|
||||
(hasEdges || hasSinks).Should().BeTrue(
|
||||
$"Target {target.FunctionName} in {Path.GetFileName(file)} should have edges or sinks");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CorpusIndex_ContainsAllGoldenSets()
|
||||
{
|
||||
// Arrange
|
||||
var indexPath = Path.Combine(_corpusPath, "corpus-index.json");
|
||||
File.Exists(indexPath).Should().BeTrue("corpus-index.json should exist");
|
||||
|
||||
var indexContent = File.ReadAllText(indexPath);
|
||||
using var doc = JsonDocument.Parse(indexContent);
|
||||
var root = doc.RootElement;
|
||||
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
// Extract IDs from filenames
|
||||
var fileIds = goldenSetFiles
|
||||
.Select(f => Path.GetFileNameWithoutExtension(f).Replace(".golden", string.Empty))
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Extract IDs from index
|
||||
var indexIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var category in root.GetProperty("categories").EnumerateObject())
|
||||
{
|
||||
foreach (var gsId in category.Value.GetProperty("golden_sets").EnumerateArray())
|
||||
{
|
||||
indexIds.Add(gsId.GetString()!);
|
||||
}
|
||||
}
|
||||
|
||||
// Assert
|
||||
foreach (var fileId in fileIds)
|
||||
{
|
||||
indexIds.Should().Contain(fileId,
|
||||
$"Golden set {fileId} from file should be listed in corpus-index.json");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CorpusIndex_TotalCountMatchesActualCount()
|
||||
{
|
||||
// Arrange
|
||||
var indexPath = Path.Combine(_corpusPath, "corpus-index.json");
|
||||
var indexContent = File.ReadAllText(indexPath);
|
||||
using var doc = JsonDocument.Parse(indexContent);
|
||||
var root = doc.RootElement;
|
||||
|
||||
var declaredTotal = root.GetProperty("total_count").GetInt32();
|
||||
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
// Assert
|
||||
declaredTotal.Should().Be(goldenSetFiles.Length,
|
||||
"total_count in index should match actual number of golden set files");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AllEdges_HaveValidFormat()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
// Act & Assert
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var definition = GoldenSetYamlSerializer.Deserialize(yaml);
|
||||
|
||||
foreach (var target in definition.Targets)
|
||||
{
|
||||
foreach (var edge in target.Edges)
|
||||
{
|
||||
// Edges should have valid from and to
|
||||
edge.From.Should().NotBeNullOrWhiteSpace(
|
||||
$"Edge in {target.FunctionName} ({Path.GetFileName(file)}) should have From");
|
||||
edge.To.Should().NotBeNullOrWhiteSpace(
|
||||
$"Edge in {target.FunctionName} ({Path.GetFileName(file)}) should have To");
|
||||
|
||||
// Basic block format validation (bb followed by number)
|
||||
edge.From.Should().MatchRegex(@"^bb\d+$",
|
||||
$"Edge From '{edge.From}' should match bbN format");
|
||||
edge.To.Should().MatchRegex(@"^bb\d+$",
|
||||
$"Edge To '{edge.To}' should match bbN format");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
235
src/__Tests/Integration/GoldenSetDiff/DeterminismTests.cs
Normal file
235
src/__Tests/Integration/GoldenSetDiff/DeterminismTests.cs
Normal file
@@ -0,0 +1,235 @@
|
||||
// Licensed under AGPL-3.0-or-later. Copyright (C) 2026 StellaOps Contributors.
|
||||
// Sprint: SPRINT_20260110_012_010_TEST
|
||||
// Task: GTV-008 - Determinism Tests
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StellaOps.BinaryIndex.GoldenSet;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Integration.GoldenSetDiff;
|
||||
|
||||
/// <summary>
|
||||
/// Tests that all golden set operations are deterministic.
|
||||
/// Same inputs must always produce same outputs.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class DeterminismTests
|
||||
{
|
||||
private readonly string _corpusPath;
|
||||
private readonly IGoldenSetValidator _validator;
|
||||
|
||||
public DeterminismTests()
|
||||
{
|
||||
var assemblyLocation = Path.GetDirectoryName(typeof(DeterminismTests).Assembly.Location)!;
|
||||
_corpusPath = Path.Combine(assemblyLocation, "golden-sets");
|
||||
|
||||
// Create validator with mocked dependencies
|
||||
var sinkRegistry = new Mock<ISinkRegistry>();
|
||||
sinkRegistry.Setup(r => r.IsKnownSink(It.IsAny<string>())).Returns(true);
|
||||
|
||||
var options = Options.Create(new GoldenSetOptions
|
||||
{
|
||||
Validation = new GoldenSetValidationOptions
|
||||
{
|
||||
OfflineMode = true,
|
||||
ValidateSinks = false,
|
||||
StrictEdgeFormat = true
|
||||
}
|
||||
});
|
||||
|
||||
_validator = new GoldenSetValidator(
|
||||
sinkRegistry.Object,
|
||||
options,
|
||||
NullLogger<GoldenSetValidator>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GoldenSetDigest_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
goldenSetFiles.Should().NotBeEmpty();
|
||||
var yaml = await File.ReadAllTextAsync(goldenSetFiles[0]);
|
||||
|
||||
// Act - compute digest multiple times
|
||||
var digests = new List<string>();
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
var result = await _validator.ValidateYamlAsync(yaml);
|
||||
if (result.IsValid && result.ContentDigest is not null)
|
||||
{
|
||||
digests.Add(result.ContentDigest);
|
||||
}
|
||||
}
|
||||
|
||||
// Assert
|
||||
digests.Should().HaveCount(5, "all validation runs should succeed");
|
||||
digests.Should().AllBeEquivalentTo(digests[0],
|
||||
"all digest computations should produce identical results");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GoldenSetSerialization_RoundTrip_PreservesContent()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
foreach (var file in goldenSetFiles.Take(5)) // Test first 5 for speed
|
||||
{
|
||||
var originalYaml = await File.ReadAllTextAsync(file);
|
||||
|
||||
// Act - deserialize then serialize
|
||||
var definition = GoldenSetYamlSerializer.Deserialize(originalYaml);
|
||||
var roundTrippedYaml = GoldenSetYamlSerializer.Serialize(definition);
|
||||
var reparsed = GoldenSetYamlSerializer.Deserialize(roundTrippedYaml);
|
||||
|
||||
// Assert - semantic equality
|
||||
reparsed.Id.Should().Be(definition.Id);
|
||||
reparsed.Component.Should().Be(definition.Component);
|
||||
reparsed.Targets.Length.Should().Be(definition.Targets.Length);
|
||||
reparsed.Metadata.AuthorId.Should().Be(definition.Metadata.AuthorId);
|
||||
reparsed.Metadata.Tags.Should().BeEquivalentTo(definition.Metadata.Tags);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GoldenSetParsing_MultipleTimes_ProducesSameResult()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
var yaml = await File.ReadAllTextAsync(goldenSetFiles[0]);
|
||||
|
||||
// Act - parse multiple times
|
||||
var definitions = new List<GoldenSetDefinition>();
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
definitions.Add(GoldenSetYamlSerializer.Deserialize(yaml));
|
||||
}
|
||||
|
||||
// Assert - all should be equivalent
|
||||
for (int i = 1; i < definitions.Count; i++)
|
||||
{
|
||||
definitions[i].Id.Should().Be(definitions[0].Id);
|
||||
definitions[i].Component.Should().Be(definitions[0].Component);
|
||||
definitions[i].Targets.Length.Should().Be(definitions[0].Targets.Length);
|
||||
|
||||
for (int j = 0; j < definitions[0].Targets.Length; j++)
|
||||
{
|
||||
definitions[i].Targets[j].FunctionName.Should().Be(definitions[0].Targets[j].FunctionName);
|
||||
definitions[i].Targets[j].Edges.Should().BeEquivalentTo(definitions[0].Targets[j].Edges);
|
||||
definitions[i].Targets[j].Sinks.Should().BeEquivalentTo(definitions[0].Targets[j].Sinks);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ContentDigest_NotAffectedByWhitespaceOnlyChanges()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
var yaml = await File.ReadAllTextAsync(goldenSetFiles[0]);
|
||||
|
||||
// Parse to get semantic content
|
||||
var definition = GoldenSetYamlSerializer.Deserialize(yaml);
|
||||
|
||||
// Re-serialize (normalizes whitespace)
|
||||
var normalizedYaml = GoldenSetYamlSerializer.Serialize(definition);
|
||||
|
||||
// Act - validate both versions
|
||||
var originalResult = await _validator.ValidateYamlAsync(yaml);
|
||||
var normalizedResult = await _validator.ValidateYamlAsync(normalizedYaml);
|
||||
|
||||
// Assert - both should have same content (digest comparison)
|
||||
// Note: digests may differ if comments are included, but semantic content should match
|
||||
originalResult.IsValid.Should().BeTrue();
|
||||
normalizedResult.IsValid.Should().BeTrue();
|
||||
|
||||
// Semantic comparison
|
||||
var originalDef = GoldenSetYamlSerializer.Deserialize(yaml);
|
||||
var normalizedDef = GoldenSetYamlSerializer.Deserialize(normalizedYaml);
|
||||
|
||||
originalDef.Id.Should().Be(normalizedDef.Id);
|
||||
originalDef.Component.Should().Be(normalizedDef.Component);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EdgeParsing_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var testEdges = new[] { "bb0->bb1", "bb3->bb7", "bb12->bb15" };
|
||||
|
||||
// Act & Assert
|
||||
foreach (var edgeStr in testEdges)
|
||||
{
|
||||
var edges = new List<BasicBlockEdge>();
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
edges.Add(BasicBlockEdge.Parse(edgeStr));
|
||||
}
|
||||
|
||||
edges.Should().AllBeEquivalentTo(edges[0],
|
||||
$"parsing '{edgeStr}' should be deterministic");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EdgeToString_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var edge = new BasicBlockEdge { From = "bb3", To = "bb7" };
|
||||
|
||||
// Act
|
||||
var strings = new List<string>();
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
strings.Add(edge.ToString());
|
||||
}
|
||||
|
||||
// Assert
|
||||
strings.Should().AllBeEquivalentTo("bb3->bb7");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AllCorpusGoldenSets_HaveStableDigests()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
var digestMap = new Dictionary<string, string>();
|
||||
|
||||
// Act - first pass: compute digests
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var result = await _validator.ValidateYamlAsync(yaml);
|
||||
if (result.IsValid && result.ContentDigest is not null)
|
||||
{
|
||||
digestMap[file] = result.ContentDigest;
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: verify digests are stable
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var result = await _validator.ValidateYamlAsync(yaml);
|
||||
|
||||
if (digestMap.TryGetValue(file, out var expectedDigest))
|
||||
{
|
||||
result.ContentDigest.Should().Be(expectedDigest,
|
||||
$"digest for {Path.GetFileName(file)} should be stable across runs");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
249
src/__Tests/Integration/GoldenSetDiff/ReplayValidationTests.cs
Normal file
249
src/__Tests/Integration/GoldenSetDiff/ReplayValidationTests.cs
Normal file
@@ -0,0 +1,249 @@
|
||||
// Licensed under AGPL-3.0-or-later. Copyright (C) 2026 StellaOps Contributors.
|
||||
// Sprint: SPRINT_20260110_012_010_TEST
|
||||
// Task: GTV-010 - Replay Validation Tests
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using StellaOps.BinaryIndex.GoldenSet;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Integration.GoldenSetDiff;
|
||||
|
||||
/// <summary>
|
||||
/// Tests that verify replay correctness - ensuring that identical inputs
|
||||
/// always produce identical outputs for audit trail validation.
|
||||
/// </summary>
|
||||
[Trait("Category", "Integration")]
|
||||
public sealed class ReplayValidationTests
|
||||
{
|
||||
private readonly string _corpusPath;
|
||||
private readonly IGoldenSetValidator _validator;
|
||||
|
||||
public ReplayValidationTests()
|
||||
{
|
||||
var assemblyLocation = Path.GetDirectoryName(typeof(ReplayValidationTests).Assembly.Location)!;
|
||||
_corpusPath = Path.Combine(assemblyLocation, "golden-sets");
|
||||
|
||||
// Create validator with mocked dependencies
|
||||
var sinkRegistry = new Mock<ISinkRegistry>();
|
||||
sinkRegistry.Setup(r => r.IsKnownSink(It.IsAny<string>())).Returns(true);
|
||||
|
||||
var options = Options.Create(new GoldenSetOptions
|
||||
{
|
||||
Validation = new GoldenSetValidationOptions
|
||||
{
|
||||
OfflineMode = true,
|
||||
ValidateSinks = false,
|
||||
StrictEdgeFormat = true
|
||||
}
|
||||
});
|
||||
|
||||
_validator = new GoldenSetValidator(
|
||||
sinkRegistry.Object,
|
||||
options,
|
||||
NullLogger<GoldenSetValidator>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Replay_GoldenSetValidation_ProducesIdenticalResult()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
goldenSetFiles.Should().NotBeEmpty();
|
||||
|
||||
foreach (var file in goldenSetFiles.Take(5)) // Test subset for speed
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
|
||||
// First validation (simulates original run)
|
||||
var originalResult = await _validator.ValidateYamlAsync(yaml);
|
||||
|
||||
// Store state for replay comparison
|
||||
var originalDigest = originalResult.ContentDigest;
|
||||
var originalIsValid = originalResult.IsValid;
|
||||
|
||||
// Replay validation (simulates later verification)
|
||||
var replayResult = await _validator.ValidateYamlAsync(yaml);
|
||||
|
||||
// Assert - replay produces identical results
|
||||
replayResult.IsValid.Should().Be(originalIsValid,
|
||||
$"Replay validation for {Path.GetFileName(file)} should produce same validity");
|
||||
replayResult.ContentDigest.Should().Be(originalDigest,
|
||||
$"Replay validation for {Path.GetFileName(file)} should produce same digest");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ContentDigest_CanBeUsed_ForReplayVerification()
|
||||
{
|
||||
// This test simulates the workflow where a digest is stored,
|
||||
// and later used to verify the golden set hasn't changed
|
||||
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
// Simulate: store digests at time T1
|
||||
var storedDigests = new Dictionary<string, string>();
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var result = await _validator.ValidateYamlAsync(yaml);
|
||||
if (result.IsValid && result.ContentDigest is not null)
|
||||
{
|
||||
storedDigests[Path.GetFileName(file)] = result.ContentDigest;
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate: verify at time T2 (same content)
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var result = await _validator.ValidateYamlAsync(yaml);
|
||||
|
||||
var fileName = Path.GetFileName(file);
|
||||
if (storedDigests.TryGetValue(fileName, out var storedDigest))
|
||||
{
|
||||
result.ContentDigest.Should().Be(storedDigest,
|
||||
$"Content digest for {fileName} should match stored value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GoldenSetModification_ChangesContentDigest()
|
||||
{
|
||||
// Arrange
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
var yaml = await File.ReadAllTextAsync(goldenSetFiles[0]);
|
||||
var original = GoldenSetYamlSerializer.Deserialize(yaml);
|
||||
|
||||
// Compute original digest
|
||||
var originalResult = await _validator.ValidateYamlAsync(yaml);
|
||||
var originalDigest = originalResult.ContentDigest;
|
||||
|
||||
// Modify the golden set (add a tag)
|
||||
var modified = original with
|
||||
{
|
||||
Metadata = original.Metadata with
|
||||
{
|
||||
Tags = original.Metadata.Tags.Add("test-modification-tag")
|
||||
}
|
||||
};
|
||||
|
||||
var modifiedYaml = GoldenSetYamlSerializer.Serialize(modified);
|
||||
var modifiedResult = await _validator.ValidateYamlAsync(modifiedYaml);
|
||||
|
||||
// Assert - modification changes digest
|
||||
modifiedResult.ContentDigest.Should().NotBe(originalDigest,
|
||||
"modifying golden set should change content digest");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ParsingOrder_DoesNotAffectDigest()
|
||||
{
|
||||
// This test verifies that the order in which golden sets are parsed
|
||||
// doesn't affect their individual digests (no cross-contamination)
|
||||
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
var digestsForwardOrder = new Dictionary<string, string>();
|
||||
var digestsReverseOrder = new Dictionary<string, string>();
|
||||
|
||||
// Parse in forward order
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var result = await _validator.ValidateYamlAsync(yaml);
|
||||
if (result.ContentDigest is not null)
|
||||
{
|
||||
digestsForwardOrder[file] = result.ContentDigest;
|
||||
}
|
||||
}
|
||||
|
||||
// Parse in reverse order
|
||||
foreach (var file in goldenSetFiles.Reverse())
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
var result = await _validator.ValidateYamlAsync(yaml);
|
||||
if (result.ContentDigest is not null)
|
||||
{
|
||||
digestsReverseOrder[file] = result.ContentDigest;
|
||||
}
|
||||
}
|
||||
|
||||
// Assert - order doesn't matter
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
if (digestsForwardOrder.TryGetValue(file, out var forwardDigest) &&
|
||||
digestsReverseOrder.TryGetValue(file, out var reverseDigest))
|
||||
{
|
||||
forwardDigest.Should().Be(reverseDigest,
|
||||
$"digest for {Path.GetFileName(file)} should be independent of parsing order");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TimestampInMetadata_DoesNotAffectSemanticDigest()
|
||||
{
|
||||
// For audit purposes, we want to ensure that semantic content
|
||||
// (excluding timestamps) can be compared
|
||||
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
var yaml = await File.ReadAllTextAsync(goldenSetFiles[0]);
|
||||
var original = GoldenSetYamlSerializer.Deserialize(yaml);
|
||||
|
||||
// Change only the reviewed_at timestamp
|
||||
var withDifferentTimestamp = original with
|
||||
{
|
||||
Metadata = original.Metadata with
|
||||
{
|
||||
ReviewedAt = DateTimeOffset.UtcNow
|
||||
}
|
||||
};
|
||||
|
||||
// Semantic comparison (excluding timestamps)
|
||||
original.Id.Should().Be(withDifferentTimestamp.Id);
|
||||
original.Component.Should().Be(withDifferentTimestamp.Component);
|
||||
original.Targets.Should().BeEquivalentTo(withDifferentTimestamp.Targets);
|
||||
original.Metadata.AuthorId.Should().Be(withDifferentTimestamp.Metadata.AuthorId);
|
||||
original.Metadata.Tags.Should().BeEquivalentTo(withDifferentTimestamp.Metadata.Tags);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EmptyAndMissingOptionalFields_HandledConsistently()
|
||||
{
|
||||
// Ensure that golden sets with and without optional fields
|
||||
// are handled consistently for replay purposes
|
||||
|
||||
var goldenSetFiles = Directory.GetFiles(
|
||||
_corpusPath, "*.golden.yaml", SearchOption.AllDirectories);
|
||||
|
||||
foreach (var file in goldenSetFiles)
|
||||
{
|
||||
var yaml = await File.ReadAllTextAsync(file);
|
||||
|
||||
// Should not throw for any optional field combinations
|
||||
var action = () => GoldenSetYamlSerializer.Deserialize(yaml);
|
||||
action.Should().NotThrow($"parsing {Path.GetFileName(file)} should handle optional fields");
|
||||
|
||||
var definition = action();
|
||||
|
||||
// Validate consistent handling of optional witness
|
||||
if (definition.Witness is not null)
|
||||
{
|
||||
definition.Witness.Arguments.IsDefaultOrEmpty.Should().Be(
|
||||
!definition.Witness.Arguments.Any() || definition.Witness.Arguments.IsDefault);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="Moq" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="coverlet.collector">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\BinaryIndex\__Libraries\StellaOps.BinaryIndex.GoldenSet\StellaOps.BinaryIndex.GoldenSet.csproj" />
|
||||
<ProjectReference Include="..\..\..\BinaryIndex\__Libraries\StellaOps.BinaryIndex.Analysis\StellaOps.BinaryIndex.Analysis.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="..\..\__Datasets\golden-sets\**\*.yaml" Link="golden-sets\%(RecursiveDir)%(FileName)%(Extension)">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="..\..\__Datasets\golden-sets\corpus-index.json" Link="golden-sets\corpus-index.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="..\..\__Datasets\binaries\**\manifest.json" Link="binaries\%(RecursiveDir)%(FileName)%(Extension)">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
Reference in New Issue
Block a user