git.stella-ops.org/tests/parity/StellaOps.Parity.Tests/ErrorModeComparisonLogic.cs

// -----------------------------------------------------------------------------
// ErrorModeComparisonLogic.cs
// Sprint: SPRINT_5100_0008_0001_competitor_parity
// Task: PARITY-5100-007 - Implement error mode comparison
// Description: Logic for comparing failure behavior between scanners
// -----------------------------------------------------------------------------

namespace StellaOps.Parity.Tests;

/// <summary>
/// Compares error handling behavior between scanners.
/// </summary>
public sealed class ErrorModeComparisonLogic
{
    /// <summary>
    /// Compares error behavior from multiple scanner runs with edge case inputs.
    /// </summary>
    public ErrorModeComparisonResult Compare(
        List<ErrorTestScenario> scenarios,
        Dictionary<string, List<ScannerOutput>> resultsByTool)
    {
        var result = new ErrorModeComparisonResult
        {
            TestedScenarios = scenarios.Count,
            ToolResults = new Dictionary<string, ToolErrorBehavior>()
        };

        foreach (var (toolName, outputs) in resultsByTool)
        {
            var behavior = new ToolErrorBehavior
            {
                ToolName = toolName,
                ScenarioResults = new List<ScenarioResult>()
            };

            for (int i = 0; i < scenarios.Count && i < outputs.Count; i++)
            {
                var scenario = scenarios[i];
                var output = outputs[i];

                var scenarioResult = new ScenarioResult
                {
                    ScenarioName = scenario.Name,
                    ExpectedBehavior = scenario.ExpectedBehavior,
                    ActualExitCode = output.ExitCode,
                    ActualSuccess = output.Success,
                    ErrorMessage = output.Error,
                    DurationMs = output.DurationMs
                };

                // Evaluate if behavior matches expectations
                scenarioResult.BehaviorMatched = EvaluateBehavior(scenario, output);
                behavior.ScenarioResults.Add(scenarioResult);
            }

            behavior.PassRate = behavior.ScenarioResults.Count > 0
                ? (double)behavior.ScenarioResults.Count(r => r.BehaviorMatched) / behavior.ScenarioResults.Count * 100
                : 0;

            result.ToolResults[toolName] = behavior;
        }

        result.Success = true;
        return result;
    }

    /// <summary>
    /// Gets predefined error test scenarios.
    /// </summary>
    public static List<ErrorTestScenario> GetErrorTestScenarios()
    {
        return
        [
            new ErrorTestScenario
            {
                Name = "malformed-image-ref",
                ImageRef = ":::invalid:::",
                Description = "Completely invalid image reference format",
                ExpectedBehavior = ExpectedErrorBehavior.GracefulError,
                ExpectedExitCode = 1
            },
            new ErrorTestScenario
            {
                Name = "nonexistent-image",
                ImageRef = "nonexistent/image:doesnotexist12345",
                Description = "Image that does not exist in any registry",
                ExpectedBehavior = ExpectedErrorBehavior.GracefulError,
                ExpectedExitCode = 1
            },
            new ErrorTestScenario
            {
                Name = "network-timeout",
                ImageRef = "10.255.255.1/timeout:latest",
                Description = "Image from unreachable network (simulates timeout)",
                ExpectedBehavior = ExpectedErrorBehavior.TimeoutOrError,
                ExpectedExitCode = 1,
                TimeoutSeconds = 30
            },
            new ErrorTestScenario
            {
                Name = "empty-image",
                ImageRef = "busybox:latest",
                Description = "Minimal image with almost no packages",
                ExpectedBehavior = ExpectedErrorBehavior.SuccessEmpty,
                ExpectedExitCode = 0
            },
            new ErrorTestScenario
            {
                Name = "scratch-image",
                ImageRef = "scratch",
                Description = "Empty scratch image",
                ExpectedBehavior = ExpectedErrorBehavior.SuccessOrError,
                ExpectedExitCode = null // Either success or graceful error
            },
            new ErrorTestScenario
            {
                Name = "large-image",
                ImageRef = "nvidia/cuda:12.3.1-devel-ubuntu22.04",
                Description = "Large image (~5GB) to test memory handling",
                ExpectedBehavior = ExpectedErrorBehavior.SuccessOrSkip,
                ExpectedExitCode = 0,
                TimeoutSeconds = 600
            },
            new ErrorTestScenario
            {
                Name = "corrupted-layers",
                ImageRef = "corrupted/test:v1",
                Description = "Image with corrupted layer data (if available)",
                ExpectedBehavior = ExpectedErrorBehavior.GracefulError,
                ExpectedExitCode = 1
            }
        ];
    }

    private static bool EvaluateBehavior(ErrorTestScenario scenario, ScannerOutput output)
    {
        return scenario.ExpectedBehavior switch
        {
            ExpectedErrorBehavior.GracefulError =>
                !output.Success && output.ExitCode != 0 && !string.IsNullOrEmpty(output.Error),

            ExpectedErrorBehavior.SuccessEmpty =>
                output.Success && output.ExitCode == 0,

            ExpectedErrorBehavior.SuccessOrError =>
                output.ExitCode == 0 || (!output.Success && !string.IsNullOrEmpty(output.Error)),

            ExpectedErrorBehavior.SuccessOrSkip =>
                output.ExitCode == 0 || output.Error?.Contains("skip", StringComparison.OrdinalIgnoreCase) == true,

            ExpectedErrorBehavior.TimeoutOrError =>
                !output.Success,

            ExpectedErrorBehavior.Crash =>
                output.ExitCode < 0 || output.ExitCode > 128,

            _ => false
        };
    }
}

/// <summary>
/// Result of error mode comparison.
/// </summary>
public sealed class ErrorModeComparisonResult
{
    public bool Success { get; set; }
    public string? Error { get; set; }
    public int TestedScenarios { get; set; }
    public Dictionary<string, ToolErrorBehavior> ToolResults { get; set; } = new();
}

/// <summary>
/// Error handling behavior for a single tool.
/// </summary>
public sealed class ToolErrorBehavior
{
    public required string ToolName { get; init; }
    public List<ScenarioResult> ScenarioResults { get; set; } = [];
    public double PassRate { get; set; }
}

/// <summary>
/// Result for a single error scenario.
/// </summary>
public sealed class ScenarioResult
{
    public required string ScenarioName { get; init; }
    public ExpectedErrorBehavior ExpectedBehavior { get; set; }
    public int ActualExitCode { get; set; }
    public bool ActualSuccess { get; set; }
    public string? ErrorMessage { get; set; }
    public long DurationMs { get; set; }
    public bool BehaviorMatched { get; set; }
}

/// <summary>
/// Defines an error test scenario.
/// </summary>
public sealed class ErrorTestScenario
{
    public required string Name { get; init; }
    public required string ImageRef { get; init; }
    public required string Description { get; init; }
    public ExpectedErrorBehavior ExpectedBehavior { get; init; }
    public int? ExpectedExitCode { get; init; }
    public int TimeoutSeconds { get; init; } = 60;
}

/// <summary>
/// Expected error behavior categories.
/// </summary>
public enum ExpectedErrorBehavior
{
    /// <summary>Tool should exit with non-zero and meaningful error message.</summary>
    GracefulError,

    /// <summary>Tool should succeed but produce empty/minimal output.</summary>
    SuccessEmpty,

    /// <summary>Either success or graceful error is acceptable.</summary>
    SuccessOrError,

    /// <summary>Either success or skip message is acceptable.</summary>
    SuccessOrSkip,

    /// <summary>Timeout or error is expected (network unreachable).</summary>
    TimeoutOrError,

    /// <summary>Tool is expected to crash (for negative testing).</summary>
    Crash
}