sprints work
This commit is contained in:
StellaOps Bot
473
tests/integration/StellaOps.Integration.E2E/ManifestComparer.cs
Normal file
473
tests/integration/StellaOps.Integration.E2E/ManifestComparer.cs
Normal file
@@ -0,0 +1,473 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ManifestComparer.cs
|
||||
// Sprint: SPRINT_8200_0001_0004_e2e_reproducibility_test
|
||||
// Task: E2E-8200-004 - Add helper to compare verdict manifests byte-for-byte
|
||||
// Description: Provides byte-for-byte comparison of manifests and detailed diff reporting.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace StellaOps.Integration.E2E;
|
||||
|
||||
/// <summary>
|
||||
/// Compares manifests and pipeline results byte-for-byte for reproducibility verification.
|
||||
/// </summary>
|
||||
public static class ManifestComparer
|
||||
{
|
||||
/// <summary>
|
||||
/// Compares two pipeline results for exact equality.
|
||||
/// </summary>
|
||||
public static ManifestComparisonResult Compare(PipelineResult expected, PipelineResult actual)
|
||||
{
|
||||
var differences = new List<ManifestDifference>();
|
||||
|
||||
// Compare verdict IDs
|
||||
if (!string.Equals(expected.VerdictId, actual.VerdictId, StringComparison.Ordinal))
|
||||
{
|
||||
differences.Add(new ManifestDifference(
|
||||
"VerdictId",
|
||||
expected.VerdictId,
|
||||
actual.VerdictId,
|
||||
DifferenceType.ValueMismatch));
|
||||
}
|
||||
|
||||
// Compare verdict hashes
|
||||
if (!string.Equals(expected.VerdictHash, actual.VerdictHash, StringComparison.Ordinal))
|
||||
{
|
||||
differences.Add(new ManifestDifference(
|
||||
"VerdictHash",
|
||||
expected.VerdictHash,
|
||||
actual.VerdictHash,
|
||||
DifferenceType.HashMismatch));
|
||||
}
|
||||
|
||||
// Compare envelope hashes
|
||||
if (!string.Equals(expected.EnvelopeHash, actual.EnvelopeHash, StringComparison.Ordinal))
|
||||
{
|
||||
differences.Add(new ManifestDifference(
|
||||
"EnvelopeHash",
|
||||
expected.EnvelopeHash,
|
||||
actual.EnvelopeHash,
|
||||
DifferenceType.HashMismatch));
|
||||
}
|
||||
|
||||
// Compare bundle manifest hashes
|
||||
if (!string.Equals(expected.BundleManifestHash, actual.BundleManifestHash, StringComparison.Ordinal))
|
||||
{
|
||||
differences.Add(new ManifestDifference(
|
||||
"BundleManifestHash",
|
||||
expected.BundleManifestHash,
|
||||
actual.BundleManifestHash,
|
||||
DifferenceType.HashMismatch));
|
||||
}
|
||||
|
||||
// Compare bundle manifest bytes
|
||||
if (!expected.BundleManifest.AsSpan().SequenceEqual(actual.BundleManifest))
|
||||
{
|
||||
var byteDiff = FindByteDifference(expected.BundleManifest, actual.BundleManifest);
|
||||
differences.Add(new ManifestDifference(
|
||||
"BundleManifest",
|
||||
$"Bytes differ at offset {byteDiff.Offset}: expected 0x{byteDiff.Expected:X2}, actual 0x{byteDiff.Actual:X2}",
|
||||
$"Expected length: {expected.BundleManifest.Length}, Actual length: {actual.BundleManifest.Length}",
|
||||
DifferenceType.ByteMismatch));
|
||||
}
|
||||
|
||||
// Compare timestamps
|
||||
if (expected.ExecutionTimestamp != actual.ExecutionTimestamp)
|
||||
{
|
||||
differences.Add(new ManifestDifference(
|
||||
"ExecutionTimestamp",
|
||||
expected.ExecutionTimestamp.ToString("O"),
|
||||
actual.ExecutionTimestamp.ToString("O"),
|
||||
DifferenceType.ValueMismatch));
|
||||
}
|
||||
|
||||
return new ManifestComparisonResult(differences.Count == 0, differences);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares multiple pipeline results to verify they are all identical.
|
||||
/// </summary>
|
||||
public static MultipleComparisonResult CompareMultiple(IReadOnlyList<PipelineResult> results)
|
||||
{
|
||||
if (results.Count == 0)
|
||||
{
|
||||
return new MultipleComparisonResult(true, [], "No results to compare");
|
||||
}
|
||||
|
||||
if (results.Count == 1)
|
||||
{
|
||||
return new MultipleComparisonResult(true, [], "Only one result, nothing to compare");
|
||||
}
|
||||
|
||||
var baseline = results[0];
|
||||
var comparisons = new List<(int Index, ManifestComparisonResult Result)>();
|
||||
var allMatch = true;
|
||||
|
||||
for (int i = 1; i < results.Count; i++)
|
||||
{
|
||||
var comparison = Compare(baseline, results[i]);
|
||||
comparisons.Add((i, comparison));
|
||||
|
||||
if (!comparison.IsMatch)
|
||||
{
|
||||
allMatch = false;
|
||||
}
|
||||
}
|
||||
|
||||
var summary = allMatch
|
||||
? $"All {results.Count} results are identical"
|
||||
: $"{comparisons.Count(c => !c.Result.IsMatch)} of {results.Count - 1} comparisons have differences";
|
||||
|
||||
return new MultipleComparisonResult(allMatch, comparisons, summary);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares two byte arrays and returns detailed difference information.
|
||||
/// </summary>
|
||||
public static ByteComparisonResult CompareBytes(ReadOnlySpan<byte> expected, ReadOnlySpan<byte> actual)
|
||||
{
|
||||
var differences = new List<ByteDifference>();
|
||||
|
||||
var minLength = Math.Min(expected.Length, actual.Length);
|
||||
var maxLength = Math.Max(expected.Length, actual.Length);
|
||||
|
||||
// Compare common bytes
|
||||
for (int i = 0; i < minLength; i++)
|
||||
{
|
||||
if (expected[i] != actual[i])
|
||||
{
|
||||
differences.Add(new ByteDifference(i, expected[i], actual[i]));
|
||||
}
|
||||
}
|
||||
|
||||
// Check for length mismatch
|
||||
var lengthMismatch = expected.Length != actual.Length;
|
||||
if (lengthMismatch)
|
||||
{
|
||||
for (int i = minLength; i < maxLength; i++)
|
||||
{
|
||||
var expectedByte = i < expected.Length ? expected[i] : (byte?)null;
|
||||
var actualByte = i < actual.Length ? actual[i] : (byte?)null;
|
||||
differences.Add(new ByteDifference(i, expectedByte, actualByte));
|
||||
}
|
||||
}
|
||||
|
||||
return new ByteComparisonResult(
|
||||
IsMatch: differences.Count == 0,
|
||||
ExpectedLength: expected.Length,
|
||||
ActualLength: actual.Length,
|
||||
Differences: differences,
|
||||
FirstDifferenceOffset: differences.Count > 0 ? differences[0].Offset : null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares two JSON documents for semantic equality (ignoring whitespace differences).
|
||||
/// </summary>
|
||||
public static JsonComparisonResult CompareJson(ReadOnlySpan<byte> expected, ReadOnlySpan<byte> actual)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var expectedDoc = JsonDocument.Parse(expected.ToArray());
|
||||
using var actualDoc = JsonDocument.Parse(actual.ToArray());
|
||||
|
||||
var differences = CompareJsonElements("$", expectedDoc.RootElement, actualDoc.RootElement);
|
||||
|
||||
return new JsonComparisonResult(
|
||||
IsMatch: differences.Count == 0,
|
||||
Differences: differences,
|
||||
ExpectedJson: Encoding.UTF8.GetString(expected),
|
||||
ActualJson: Encoding.UTF8.GetString(actual));
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
return new JsonComparisonResult(
|
||||
IsMatch: false,
|
||||
Differences: [new JsonDifference("$", $"JSON parse error: {ex.Message}", null, JsonDifferenceType.ParseError)],
|
||||
ExpectedJson: Encoding.UTF8.GetString(expected),
|
||||
ActualJson: Encoding.UTF8.GetString(actual));
|
||||
}
|
||||
}
|
||||
|
||||
private static List<JsonDifference> CompareJsonElements(string path, JsonElement expected, JsonElement actual)
|
||||
{
|
||||
var differences = new List<JsonDifference>();
|
||||
|
||||
if (expected.ValueKind != actual.ValueKind)
|
||||
{
|
||||
differences.Add(new JsonDifference(
|
||||
path,
|
||||
$"Type: {expected.ValueKind}",
|
||||
$"Type: {actual.ValueKind}",
|
||||
JsonDifferenceType.TypeMismatch));
|
||||
return differences;
|
||||
}
|
||||
|
||||
switch (expected.ValueKind)
|
||||
{
|
||||
case JsonValueKind.Object:
|
||||
var expectedProps = expected.EnumerateObject().ToDictionary(p => p.Name);
|
||||
var actualProps = actual.EnumerateObject().ToDictionary(p => p.Name);
|
||||
|
||||
foreach (var prop in expectedProps)
|
||||
{
|
||||
var propPath = $"{path}.{prop.Key}";
|
||||
if (!actualProps.TryGetValue(prop.Key, out var actualProp))
|
||||
{
|
||||
differences.Add(new JsonDifference(propPath, prop.Value.ToString(), null, JsonDifferenceType.MissingProperty));
|
||||
}
|
||||
else
|
||||
{
|
||||
differences.AddRange(CompareJsonElements(propPath, prop.Value.Value, actualProp.Value));
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var prop in actualProps)
|
||||
{
|
||||
if (!expectedProps.ContainsKey(prop.Key))
|
||||
{
|
||||
var propPath = $"{path}.{prop.Key}";
|
||||
differences.Add(new JsonDifference(propPath, null, prop.Value.ToString(), JsonDifferenceType.ExtraProperty));
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case JsonValueKind.Array:
|
||||
var expectedArray = expected.EnumerateArray().ToList();
|
||||
var actualArray = actual.EnumerateArray().ToList();
|
||||
|
||||
if (expectedArray.Count != actualArray.Count)
|
||||
{
|
||||
differences.Add(new JsonDifference(
|
||||
path,
|
||||
$"Length: {expectedArray.Count}",
|
||||
$"Length: {actualArray.Count}",
|
||||
JsonDifferenceType.ArrayLengthMismatch));
|
||||
}
|
||||
|
||||
var minCount = Math.Min(expectedArray.Count, actualArray.Count);
|
||||
for (int i = 0; i < minCount; i++)
|
||||
{
|
||||
differences.AddRange(CompareJsonElements($"{path}[{i}]", expectedArray[i], actualArray[i]));
|
||||
}
|
||||
break;
|
||||
|
||||
case JsonValueKind.String:
|
||||
if (expected.GetString() != actual.GetString())
|
||||
{
|
||||
differences.Add(new JsonDifference(path, expected.GetString(), actual.GetString(), JsonDifferenceType.ValueMismatch));
|
||||
}
|
||||
break;
|
||||
|
||||
case JsonValueKind.Number:
|
||||
if (expected.GetRawText() != actual.GetRawText())
|
||||
{
|
||||
differences.Add(new JsonDifference(path, expected.GetRawText(), actual.GetRawText(), JsonDifferenceType.ValueMismatch));
|
||||
}
|
||||
break;
|
||||
|
||||
case JsonValueKind.True:
|
||||
case JsonValueKind.False:
|
||||
if (expected.GetBoolean() != actual.GetBoolean())
|
||||
{
|
||||
differences.Add(new JsonDifference(path, expected.GetBoolean().ToString(), actual.GetBoolean().ToString(), JsonDifferenceType.ValueMismatch));
|
||||
}
|
||||
break;
|
||||
|
||||
case JsonValueKind.Null:
|
||||
// Both are null, no difference
|
||||
break;
|
||||
}
|
||||
|
||||
return differences;
|
||||
}
|
||||
|
||||
private static ByteDifference FindByteDifference(byte[] expected, byte[] actual)
|
||||
{
|
||||
var minLength = Math.Min(expected.Length, actual.Length);
|
||||
|
||||
for (int i = 0; i < minLength; i++)
|
||||
{
|
||||
if (expected[i] != actual[i])
|
||||
{
|
||||
return new ByteDifference(i, expected[i], actual[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Length difference
|
||||
if (expected.Length != actual.Length)
|
||||
{
|
||||
return new ByteDifference(
|
||||
minLength,
|
||||
minLength < expected.Length ? expected[minLength] : (byte?)null,
|
||||
minLength < actual.Length ? actual[minLength] : (byte?)null);
|
||||
}
|
||||
|
||||
// No difference (shouldn't happen if called correctly)
|
||||
return new ByteDifference(0, 0, 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a detailed diff report for debugging reproducibility failures.
|
||||
/// </summary>
|
||||
public static string GenerateDiffReport(ManifestComparisonResult comparison)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
sb.AppendLine("=== Manifest Comparison Report ===");
|
||||
sb.AppendLine();
|
||||
|
||||
if (comparison.IsMatch)
|
||||
{
|
||||
sb.AppendLine("✓ All fields match exactly");
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
sb.AppendLine($"✗ Found {comparison.Differences.Count} difference(s):");
|
||||
sb.AppendLine();
|
||||
|
||||
foreach (var diff in comparison.Differences)
|
||||
{
|
||||
sb.AppendLine($" [{diff.Type}] {diff.Field}:");
|
||||
sb.AppendLine($" Expected: {diff.Expected}");
|
||||
sb.AppendLine($" Actual: {diff.Actual}");
|
||||
sb.AppendLine();
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates a hex dump comparison for byte-level debugging.
|
||||
/// </summary>
|
||||
public static string GenerateHexDump(ReadOnlySpan<byte> expected, ReadOnlySpan<byte> actual, int contextBytes = 16)
|
||||
{
|
||||
var comparison = CompareBytes(expected, actual);
|
||||
var sb = new StringBuilder();
|
||||
|
||||
sb.AppendLine("=== Hex Dump Comparison ===");
|
||||
sb.AppendLine($"Expected length: {expected.Length}");
|
||||
sb.AppendLine($"Actual length: {actual.Length}");
|
||||
sb.AppendLine();
|
||||
|
||||
if (comparison.IsMatch)
|
||||
{
|
||||
sb.AppendLine("✓ Bytes are identical");
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
sb.AppendLine($"✗ Found {comparison.Differences.Count} byte difference(s)");
|
||||
sb.AppendLine();
|
||||
|
||||
// Show first few differences with context
|
||||
var diffsToShow = comparison.Differences.Take(5).ToList();
|
||||
foreach (var diff in diffsToShow)
|
||||
{
|
||||
var startOffset = Math.Max(0, diff.Offset - contextBytes);
|
||||
var endOffset = Math.Min(Math.Max(expected.Length, actual.Length), diff.Offset + contextBytes);
|
||||
|
||||
sb.AppendLine($"Difference at offset 0x{diff.Offset:X8} ({diff.Offset}):");
|
||||
sb.AppendLine($" Expected: 0x{diff.Expected:X2} ('{(char?)(diff.Expected >= 32 && diff.Expected < 127 ? diff.Expected : '.')}')" );
|
||||
sb.AppendLine($" Actual: 0x{diff.Actual:X2} ('{(char?)(diff.Actual >= 32 && diff.Actual < 127 ? diff.Actual : '.')}')" );
|
||||
sb.AppendLine();
|
||||
}
|
||||
|
||||
if (comparison.Differences.Count > 5)
|
||||
{
|
||||
sb.AppendLine($"... and {comparison.Differences.Count - 5} more differences");
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
#region Result Types
|
||||
|
||||
/// <summary>
|
||||
/// Result of comparing two manifests.
|
||||
/// </summary>
|
||||
public sealed record ManifestComparisonResult(
|
||||
bool IsMatch,
|
||||
IReadOnlyList<ManifestDifference> Differences);
|
||||
|
||||
/// <summary>
|
||||
/// A single difference between manifests.
|
||||
/// </summary>
|
||||
public sealed record ManifestDifference(
|
||||
string Field,
|
||||
string? Expected,
|
||||
string? Actual,
|
||||
DifferenceType Type);
|
||||
|
||||
/// <summary>
|
||||
/// Type of difference found.
|
||||
/// </summary>
|
||||
public enum DifferenceType
|
||||
{
|
||||
ValueMismatch,
|
||||
HashMismatch,
|
||||
ByteMismatch,
|
||||
LengthMismatch,
|
||||
Missing,
|
||||
Extra
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of comparing multiple pipeline results.
|
||||
/// </summary>
|
||||
public sealed record MultipleComparisonResult(
|
||||
bool AllMatch,
|
||||
IReadOnlyList<(int Index, ManifestComparisonResult Result)> Comparisons,
|
||||
string Summary);
|
||||
|
||||
/// <summary>
|
||||
/// Result of byte-level comparison.
|
||||
/// </summary>
|
||||
public sealed record ByteComparisonResult(
|
||||
bool IsMatch,
|
||||
int ExpectedLength,
|
||||
int ActualLength,
|
||||
IReadOnlyList<ByteDifference> Differences,
|
||||
int? FirstDifferenceOffset);
|
||||
|
||||
/// <summary>
|
||||
/// A single byte difference.
|
||||
/// </summary>
|
||||
public sealed record ByteDifference(
|
||||
int Offset,
|
||||
byte? Expected,
|
||||
byte? Actual);
|
||||
|
||||
/// <summary>
|
||||
/// Result of JSON comparison.
|
||||
/// </summary>
|
||||
public sealed record JsonComparisonResult(
|
||||
bool IsMatch,
|
||||
IReadOnlyList<JsonDifference> Differences,
|
||||
string ExpectedJson,
|
||||
string ActualJson);
|
||||
|
||||
/// <summary>
|
||||
/// A single JSON difference.
|
||||
/// </summary>
|
||||
public sealed record JsonDifference(
|
||||
string Path,
|
||||
string? Expected,
|
||||
string? Actual,
|
||||
JsonDifferenceType Type);
|
||||
|
||||
/// <summary>
|
||||
/// Type of JSON difference.
|
||||
/// </summary>
|
||||
public enum JsonDifferenceType
|
||||
{
|
||||
ValueMismatch,
|
||||
TypeMismatch,
|
||||
MissingProperty,
|
||||
ExtraProperty,
|
||||
ArrayLengthMismatch,
|
||||
ParseError
|
||||
}
|
||||
|
||||
#endregion
|
||||
Reference in New Issue
Block a user