// ----------------------------------------------------------------------------- // ErrorModeComparisonLogic.cs // Sprint: SPRINT_5100_0008_0001_competitor_parity // Task: PARITY-5100-007 - Implement error mode comparison // Description: Logic for comparing failure behavior between scanners // ----------------------------------------------------------------------------- namespace StellaOps.Parity.Tests; /// /// Compares error handling behavior between scanners. /// public sealed class ErrorModeComparisonLogic { /// /// Compares error behavior from multiple scanner runs with edge case inputs. /// public ErrorModeComparisonResult Compare( List scenarios, Dictionary> resultsByTool) { var result = new ErrorModeComparisonResult { TestedScenarios = scenarios.Count, ToolResults = new Dictionary() }; foreach (var (toolName, outputs) in resultsByTool) { var behavior = new ToolErrorBehavior { ToolName = toolName, ScenarioResults = new List() }; for (int i = 0; i < scenarios.Count && i < outputs.Count; i++) { var scenario = scenarios[i]; var output = outputs[i]; var scenarioResult = new ScenarioResult { ScenarioName = scenario.Name, ExpectedBehavior = scenario.ExpectedBehavior, ActualExitCode = output.ExitCode, ActualSuccess = output.Success, ErrorMessage = output.Error, DurationMs = output.DurationMs }; // Evaluate if behavior matches expectations scenarioResult.BehaviorMatched = EvaluateBehavior(scenario, output); behavior.ScenarioResults.Add(scenarioResult); } behavior.PassRate = behavior.ScenarioResults.Count > 0 ? (double)behavior.ScenarioResults.Count(r => r.BehaviorMatched) / behavior.ScenarioResults.Count * 100 : 0; result.ToolResults[toolName] = behavior; } result.Success = true; return result; } /// /// Gets predefined error test scenarios. /// public static List GetErrorTestScenarios() { return [ new ErrorTestScenario { Name = "malformed-image-ref", ImageRef = ":::invalid:::", Description = "Completely invalid image reference format", ExpectedBehavior = ExpectedErrorBehavior.GracefulError, ExpectedExitCode = 1 }, new ErrorTestScenario { Name = "nonexistent-image", ImageRef = "nonexistent/image:doesnotexist12345", Description = "Image that does not exist in any registry", ExpectedBehavior = ExpectedErrorBehavior.GracefulError, ExpectedExitCode = 1 }, new ErrorTestScenario { Name = "network-timeout", ImageRef = "10.255.255.1/timeout:latest", Description = "Image from unreachable network (simulates timeout)", ExpectedBehavior = ExpectedErrorBehavior.TimeoutOrError, ExpectedExitCode = 1, TimeoutSeconds = 30 }, new ErrorTestScenario { Name = "empty-image", ImageRef = "busybox:latest", Description = "Minimal image with almost no packages", ExpectedBehavior = ExpectedErrorBehavior.SuccessEmpty, ExpectedExitCode = 0 }, new ErrorTestScenario { Name = "scratch-image", ImageRef = "scratch", Description = "Empty scratch image", ExpectedBehavior = ExpectedErrorBehavior.SuccessOrError, ExpectedExitCode = null // Either success or graceful error }, new ErrorTestScenario { Name = "large-image", ImageRef = "nvidia/cuda:12.3.1-devel-ubuntu22.04", Description = "Large image (~5GB) to test memory handling", ExpectedBehavior = ExpectedErrorBehavior.SuccessOrSkip, ExpectedExitCode = 0, TimeoutSeconds = 600 }, new ErrorTestScenario { Name = "corrupted-layers", ImageRef = "corrupted/test:v1", Description = "Image with corrupted layer data (if available)", ExpectedBehavior = ExpectedErrorBehavior.GracefulError, ExpectedExitCode = 1 } ]; } private static bool EvaluateBehavior(ErrorTestScenario scenario, ScannerOutput output) { return scenario.ExpectedBehavior switch { ExpectedErrorBehavior.GracefulError => !output.Success && output.ExitCode != 0 && !string.IsNullOrEmpty(output.Error), ExpectedErrorBehavior.SuccessEmpty => output.Success && output.ExitCode == 0, ExpectedErrorBehavior.SuccessOrError => output.ExitCode == 0 || (!output.Success && !string.IsNullOrEmpty(output.Error)), ExpectedErrorBehavior.SuccessOrSkip => output.ExitCode == 0 || output.Error?.Contains("skip", StringComparison.OrdinalIgnoreCase) == true, ExpectedErrorBehavior.TimeoutOrError => !output.Success, ExpectedErrorBehavior.Crash => output.ExitCode < 0 || output.ExitCode > 128, _ => false }; } } /// /// Result of error mode comparison. /// public sealed class ErrorModeComparisonResult { public bool Success { get; set; } public string? Error { get; set; } public int TestedScenarios { get; set; } public Dictionary ToolResults { get; set; } = new(); } /// /// Error handling behavior for a single tool. /// public sealed class ToolErrorBehavior { public required string ToolName { get; init; } public List ScenarioResults { get; set; } = []; public double PassRate { get; set; } } /// /// Result for a single error scenario. /// public sealed class ScenarioResult { public required string ScenarioName { get; init; } public ExpectedErrorBehavior ExpectedBehavior { get; set; } public int ActualExitCode { get; set; } public bool ActualSuccess { get; set; } public string? ErrorMessage { get; set; } public long DurationMs { get; set; } public bool BehaviorMatched { get; set; } } /// /// Defines an error test scenario. /// public sealed class ErrorTestScenario { public required string Name { get; init; } public required string ImageRef { get; init; } public required string Description { get; init; } public ExpectedErrorBehavior ExpectedBehavior { get; init; } public int? ExpectedExitCode { get; init; } public int TimeoutSeconds { get; init; } = 60; } /// /// Expected error behavior categories. /// public enum ExpectedErrorBehavior { /// Tool should exit with non-zero and meaningful error message. GracefulError, /// Tool should succeed but produce empty/minimal output. SuccessEmpty, /// Either success or graceful error is acceptable. SuccessOrError, /// Either success or skip message is acceptable. SuccessOrSkip, /// Timeout or error is expected (network unreachable). TimeoutOrError, /// Tool is expected to crash (for negative testing). Crash }