// ----------------------------------------------------------------------------- // ManifestComparer.cs // Sprint: SPRINT_8200_0001_0004_e2e_reproducibility_test // Task: E2E-8200-004 - Add helper to compare verdict manifests byte-for-byte // Description: Provides byte-for-byte comparison of manifests and detailed diff reporting. // ----------------------------------------------------------------------------- using System.Security.Cryptography; using System.Text; using System.Text.Json; namespace StellaOps.Integration.E2E; /// /// Compares manifests and pipeline results byte-for-byte for reproducibility verification. /// public static class ManifestComparer { /// /// Compares two pipeline results for exact equality. /// public static ManifestComparisonResult Compare(PipelineResult expected, PipelineResult actual) { var differences = new List(); // Compare verdict IDs if (!string.Equals(expected.VerdictId, actual.VerdictId, StringComparison.Ordinal)) { differences.Add(new ManifestDifference( "VerdictId", expected.VerdictId, actual.VerdictId, DifferenceType.ValueMismatch)); } // Compare verdict hashes if (!string.Equals(expected.VerdictHash, actual.VerdictHash, StringComparison.Ordinal)) { differences.Add(new ManifestDifference( "VerdictHash", expected.VerdictHash, actual.VerdictHash, DifferenceType.HashMismatch)); } // Compare envelope hashes if (!string.Equals(expected.EnvelopeHash, actual.EnvelopeHash, StringComparison.Ordinal)) { differences.Add(new ManifestDifference( "EnvelopeHash", expected.EnvelopeHash, actual.EnvelopeHash, DifferenceType.HashMismatch)); } // Compare bundle manifest hashes if (!string.Equals(expected.BundleManifestHash, actual.BundleManifestHash, StringComparison.Ordinal)) { differences.Add(new ManifestDifference( "BundleManifestHash", expected.BundleManifestHash, actual.BundleManifestHash, DifferenceType.HashMismatch)); } // Compare bundle manifest bytes if (!expected.BundleManifest.AsSpan().SequenceEqual(actual.BundleManifest)) { var byteDiff = FindByteDifference(expected.BundleManifest, actual.BundleManifest); differences.Add(new ManifestDifference( "BundleManifest", $"Bytes differ at offset {byteDiff.Offset}: expected 0x{byteDiff.Expected:X2}, actual 0x{byteDiff.Actual:X2}", $"Expected length: {expected.BundleManifest.Length}, Actual length: {actual.BundleManifest.Length}", DifferenceType.ByteMismatch)); } // Compare timestamps if (expected.ExecutionTimestamp != actual.ExecutionTimestamp) { differences.Add(new ManifestDifference( "ExecutionTimestamp", expected.ExecutionTimestamp.ToString("O"), actual.ExecutionTimestamp.ToString("O"), DifferenceType.ValueMismatch)); } return new ManifestComparisonResult(differences.Count == 0, differences); } /// /// Compares multiple pipeline results to verify they are all identical. /// public static MultipleComparisonResult CompareMultiple(IReadOnlyList results) { if (results.Count == 0) { return new MultipleComparisonResult(true, [], "No results to compare"); } if (results.Count == 1) { return new MultipleComparisonResult(true, [], "Only one result, nothing to compare"); } var baseline = results[0]; var comparisons = new List<(int Index, ManifestComparisonResult Result)>(); var allMatch = true; for (int i = 1; i < results.Count; i++) { var comparison = Compare(baseline, results[i]); comparisons.Add((i, comparison)); if (!comparison.IsMatch) { allMatch = false; } } var summary = allMatch ? $"All {results.Count} results are identical" : $"{comparisons.Count(c => !c.Result.IsMatch)} of {results.Count - 1} comparisons have differences"; return new MultipleComparisonResult(allMatch, comparisons, summary); } /// /// Compares two byte arrays and returns detailed difference information. /// public static ByteComparisonResult CompareBytes(ReadOnlySpan expected, ReadOnlySpan actual) { var differences = new List(); var minLength = Math.Min(expected.Length, actual.Length); var maxLength = Math.Max(expected.Length, actual.Length); // Compare common bytes for (int i = 0; i < minLength; i++) { if (expected[i] != actual[i]) { differences.Add(new ByteDifference(i, expected[i], actual[i])); } } // Check for length mismatch var lengthMismatch = expected.Length != actual.Length; if (lengthMismatch) { for (int i = minLength; i < maxLength; i++) { var expectedByte = i < expected.Length ? expected[i] : (byte?)null; var actualByte = i < actual.Length ? actual[i] : (byte?)null; differences.Add(new ByteDifference(i, expectedByte, actualByte)); } } return new ByteComparisonResult( IsMatch: differences.Count == 0, ExpectedLength: expected.Length, ActualLength: actual.Length, Differences: differences, FirstDifferenceOffset: differences.Count > 0 ? differences[0].Offset : null); } /// /// Compares two JSON documents for semantic equality (ignoring whitespace differences). /// public static JsonComparisonResult CompareJson(ReadOnlySpan expected, ReadOnlySpan actual) { try { using var expectedDoc = JsonDocument.Parse(expected.ToArray()); using var actualDoc = JsonDocument.Parse(actual.ToArray()); var differences = CompareJsonElements("$", expectedDoc.RootElement, actualDoc.RootElement); return new JsonComparisonResult( IsMatch: differences.Count == 0, Differences: differences, ExpectedJson: Encoding.UTF8.GetString(expected), ActualJson: Encoding.UTF8.GetString(actual)); } catch (JsonException ex) { return new JsonComparisonResult( IsMatch: false, Differences: [new JsonDifference("$", $"JSON parse error: {ex.Message}", null, JsonDifferenceType.ParseError)], ExpectedJson: Encoding.UTF8.GetString(expected), ActualJson: Encoding.UTF8.GetString(actual)); } } private static List CompareJsonElements(string path, JsonElement expected, JsonElement actual) { var differences = new List(); if (expected.ValueKind != actual.ValueKind) { differences.Add(new JsonDifference( path, $"Type: {expected.ValueKind}", $"Type: {actual.ValueKind}", JsonDifferenceType.TypeMismatch)); return differences; } switch (expected.ValueKind) { case JsonValueKind.Object: var expectedProps = expected.EnumerateObject().ToDictionary(p => p.Name); var actualProps = actual.EnumerateObject().ToDictionary(p => p.Name); foreach (var prop in expectedProps) { var propPath = $"{path}.{prop.Key}"; if (!actualProps.TryGetValue(prop.Key, out var actualProp)) { differences.Add(new JsonDifference(propPath, prop.Value.ToString(), null, JsonDifferenceType.MissingProperty)); } else { differences.AddRange(CompareJsonElements(propPath, prop.Value.Value, actualProp.Value)); } } foreach (var prop in actualProps) { if (!expectedProps.ContainsKey(prop.Key)) { var propPath = $"{path}.{prop.Key}"; differences.Add(new JsonDifference(propPath, null, prop.Value.ToString(), JsonDifferenceType.ExtraProperty)); } } break; case JsonValueKind.Array: var expectedArray = expected.EnumerateArray().ToList(); var actualArray = actual.EnumerateArray().ToList(); if (expectedArray.Count != actualArray.Count) { differences.Add(new JsonDifference( path, $"Length: {expectedArray.Count}", $"Length: {actualArray.Count}", JsonDifferenceType.ArrayLengthMismatch)); } var minCount = Math.Min(expectedArray.Count, actualArray.Count); for (int i = 0; i < minCount; i++) { differences.AddRange(CompareJsonElements($"{path}[{i}]", expectedArray[i], actualArray[i])); } break; case JsonValueKind.String: if (expected.GetString() != actual.GetString()) { differences.Add(new JsonDifference(path, expected.GetString(), actual.GetString(), JsonDifferenceType.ValueMismatch)); } break; case JsonValueKind.Number: if (expected.GetRawText() != actual.GetRawText()) { differences.Add(new JsonDifference(path, expected.GetRawText(), actual.GetRawText(), JsonDifferenceType.ValueMismatch)); } break; case JsonValueKind.True: case JsonValueKind.False: if (expected.GetBoolean() != actual.GetBoolean()) { differences.Add(new JsonDifference(path, expected.GetBoolean().ToString(), actual.GetBoolean().ToString(), JsonDifferenceType.ValueMismatch)); } break; case JsonValueKind.Null: // Both are null, no difference break; } return differences; } private static ByteDifference FindByteDifference(byte[] expected, byte[] actual) { var minLength = Math.Min(expected.Length, actual.Length); for (int i = 0; i < minLength; i++) { if (expected[i] != actual[i]) { return new ByteDifference(i, expected[i], actual[i]); } } // Length difference if (expected.Length != actual.Length) { return new ByteDifference( minLength, minLength < expected.Length ? expected[minLength] : (byte?)null, minLength < actual.Length ? actual[minLength] : (byte?)null); } // No difference (shouldn't happen if called correctly) return new ByteDifference(0, 0, 0); } /// /// Generates a detailed diff report for debugging reproducibility failures. /// public static string GenerateDiffReport(ManifestComparisonResult comparison) { var sb = new StringBuilder(); sb.AppendLine("=== Manifest Comparison Report ==="); sb.AppendLine(); if (comparison.IsMatch) { sb.AppendLine("✓ All fields match exactly"); return sb.ToString(); } sb.AppendLine($"✗ Found {comparison.Differences.Count} difference(s):"); sb.AppendLine(); foreach (var diff in comparison.Differences) { sb.AppendLine($" [{diff.Type}] {diff.Field}:"); sb.AppendLine($" Expected: {diff.Expected}"); sb.AppendLine($" Actual: {diff.Actual}"); sb.AppendLine(); } return sb.ToString(); } /// /// Generates a hex dump comparison for byte-level debugging. /// public static string GenerateHexDump(ReadOnlySpan expected, ReadOnlySpan actual, int contextBytes = 16) { var comparison = CompareBytes(expected, actual); var sb = new StringBuilder(); sb.AppendLine("=== Hex Dump Comparison ==="); sb.AppendLine($"Expected length: {expected.Length}"); sb.AppendLine($"Actual length: {actual.Length}"); sb.AppendLine(); if (comparison.IsMatch) { sb.AppendLine("✓ Bytes are identical"); return sb.ToString(); } sb.AppendLine($"✗ Found {comparison.Differences.Count} byte difference(s)"); sb.AppendLine(); // Show first few differences with context var diffsToShow = comparison.Differences.Take(5).ToList(); foreach (var diff in diffsToShow) { var startOffset = Math.Max(0, diff.Offset - contextBytes); var endOffset = Math.Min(Math.Max(expected.Length, actual.Length), diff.Offset + contextBytes); sb.AppendLine($"Difference at offset 0x{diff.Offset:X8} ({diff.Offset}):"); sb.AppendLine($" Expected: 0x{diff.Expected:X2} ('{(char?)(diff.Expected >= 32 && diff.Expected < 127 ? diff.Expected : '.')}')" ); sb.AppendLine($" Actual: 0x{diff.Actual:X2} ('{(char?)(diff.Actual >= 32 && diff.Actual < 127 ? diff.Actual : '.')}')" ); sb.AppendLine(); } if (comparison.Differences.Count > 5) { sb.AppendLine($"... and {comparison.Differences.Count - 5} more differences"); } return sb.ToString(); } } #region Result Types /// /// Result of comparing two manifests. /// public sealed record ManifestComparisonResult( bool IsMatch, IReadOnlyList Differences); /// /// A single difference between manifests. /// public sealed record ManifestDifference( string Field, string? Expected, string? Actual, DifferenceType Type); /// /// Type of difference found. /// public enum DifferenceType { ValueMismatch, HashMismatch, ByteMismatch, LengthMismatch, Missing, Extra } /// /// Result of comparing multiple pipeline results. /// public sealed record MultipleComparisonResult( bool AllMatch, IReadOnlyList<(int Index, ManifestComparisonResult Result)> Comparisons, string Summary); /// /// Result of byte-level comparison. /// public sealed record ByteComparisonResult( bool IsMatch, int ExpectedLength, int ActualLength, IReadOnlyList Differences, int? FirstDifferenceOffset); /// /// A single byte difference. /// public sealed record ByteDifference( int Offset, byte? Expected, byte? Actual); /// /// Result of JSON comparison. /// public sealed record JsonComparisonResult( bool IsMatch, IReadOnlyList Differences, string ExpectedJson, string ActualJson); /// /// A single JSON difference. /// public sealed record JsonDifference( string Path, string? Expected, string? Actual, JsonDifferenceType Type); /// /// Type of JSON difference. /// public enum JsonDifferenceType { ValueMismatch, TypeMismatch, MissingProperty, ExtraProperty, ArrayLengthMismatch, ParseError } #endregion