5100* tests strengthtenen work
This commit is contained in:
StellaOps Bot
232
tests/parity/StellaOps.Parity.Tests/ErrorModeComparisonLogic.cs
Normal file
232
tests/parity/StellaOps.Parity.Tests/ErrorModeComparisonLogic.cs
Normal file
@@ -0,0 +1,232 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ErrorModeComparisonLogic.cs
|
||||
// Sprint: SPRINT_5100_0008_0001_competitor_parity
|
||||
// Task: PARITY-5100-007 - Implement error mode comparison
|
||||
// Description: Logic for comparing failure behavior between scanners
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
namespace StellaOps.Parity.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Compares error handling behavior between scanners.
|
||||
/// </summary>
|
||||
public sealed class ErrorModeComparisonLogic
|
||||
{
|
||||
/// <summary>
|
||||
/// Compares error behavior from multiple scanner runs with edge case inputs.
|
||||
/// </summary>
|
||||
public ErrorModeComparisonResult Compare(
|
||||
List<ErrorTestScenario> scenarios,
|
||||
Dictionary<string, List<ScannerOutput>> resultsByTool)
|
||||
{
|
||||
var result = new ErrorModeComparisonResult
|
||||
{
|
||||
TestedScenarios = scenarios.Count,
|
||||
ToolResults = new Dictionary<string, ToolErrorBehavior>()
|
||||
};
|
||||
|
||||
foreach (var (toolName, outputs) in resultsByTool)
|
||||
{
|
||||
var behavior = new ToolErrorBehavior
|
||||
{
|
||||
ToolName = toolName,
|
||||
ScenarioResults = new List<ScenarioResult>()
|
||||
};
|
||||
|
||||
for (int i = 0; i < scenarios.Count && i < outputs.Count; i++)
|
||||
{
|
||||
var scenario = scenarios[i];
|
||||
var output = outputs[i];
|
||||
|
||||
var scenarioResult = new ScenarioResult
|
||||
{
|
||||
ScenarioName = scenario.Name,
|
||||
ExpectedBehavior = scenario.ExpectedBehavior,
|
||||
ActualExitCode = output.ExitCode,
|
||||
ActualSuccess = output.Success,
|
||||
ErrorMessage = output.Error,
|
||||
DurationMs = output.DurationMs
|
||||
};
|
||||
|
||||
// Evaluate if behavior matches expectations
|
||||
scenarioResult.BehaviorMatched = EvaluateBehavior(scenario, output);
|
||||
behavior.ScenarioResults.Add(scenarioResult);
|
||||
}
|
||||
|
||||
behavior.PassRate = behavior.ScenarioResults.Count > 0
|
||||
? (double)behavior.ScenarioResults.Count(r => r.BehaviorMatched) / behavior.ScenarioResults.Count * 100
|
||||
: 0;
|
||||
|
||||
result.ToolResults[toolName] = behavior;
|
||||
}
|
||||
|
||||
result.Success = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets predefined error test scenarios.
|
||||
/// </summary>
|
||||
public static List<ErrorTestScenario> GetErrorTestScenarios()
|
||||
{
|
||||
return
|
||||
[
|
||||
new ErrorTestScenario
|
||||
{
|
||||
Name = "malformed-image-ref",
|
||||
ImageRef = ":::invalid:::",
|
||||
Description = "Completely invalid image reference format",
|
||||
ExpectedBehavior = ExpectedErrorBehavior.GracefulError,
|
||||
ExpectedExitCode = 1
|
||||
},
|
||||
new ErrorTestScenario
|
||||
{
|
||||
Name = "nonexistent-image",
|
||||
ImageRef = "nonexistent/image:doesnotexist12345",
|
||||
Description = "Image that does not exist in any registry",
|
||||
ExpectedBehavior = ExpectedErrorBehavior.GracefulError,
|
||||
ExpectedExitCode = 1
|
||||
},
|
||||
new ErrorTestScenario
|
||||
{
|
||||
Name = "network-timeout",
|
||||
ImageRef = "10.255.255.1/timeout:latest",
|
||||
Description = "Image from unreachable network (simulates timeout)",
|
||||
ExpectedBehavior = ExpectedErrorBehavior.TimeoutOrError,
|
||||
ExpectedExitCode = 1,
|
||||
TimeoutSeconds = 30
|
||||
},
|
||||
new ErrorTestScenario
|
||||
{
|
||||
Name = "empty-image",
|
||||
ImageRef = "busybox:latest",
|
||||
Description = "Minimal image with almost no packages",
|
||||
ExpectedBehavior = ExpectedErrorBehavior.SuccessEmpty,
|
||||
ExpectedExitCode = 0
|
||||
},
|
||||
new ErrorTestScenario
|
||||
{
|
||||
Name = "scratch-image",
|
||||
ImageRef = "scratch",
|
||||
Description = "Empty scratch image",
|
||||
ExpectedBehavior = ExpectedErrorBehavior.SuccessOrError,
|
||||
ExpectedExitCode = null // Either success or graceful error
|
||||
},
|
||||
new ErrorTestScenario
|
||||
{
|
||||
Name = "large-image",
|
||||
ImageRef = "nvidia/cuda:12.3.1-devel-ubuntu22.04",
|
||||
Description = "Large image (~5GB) to test memory handling",
|
||||
ExpectedBehavior = ExpectedErrorBehavior.SuccessOrSkip,
|
||||
ExpectedExitCode = 0,
|
||||
TimeoutSeconds = 600
|
||||
},
|
||||
new ErrorTestScenario
|
||||
{
|
||||
Name = "corrupted-layers",
|
||||
ImageRef = "corrupted/test:v1",
|
||||
Description = "Image with corrupted layer data (if available)",
|
||||
ExpectedBehavior = ExpectedErrorBehavior.GracefulError,
|
||||
ExpectedExitCode = 1
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
private static bool EvaluateBehavior(ErrorTestScenario scenario, ScannerOutput output)
|
||||
{
|
||||
return scenario.ExpectedBehavior switch
|
||||
{
|
||||
ExpectedErrorBehavior.GracefulError =>
|
||||
!output.Success && output.ExitCode != 0 && !string.IsNullOrEmpty(output.Error),
|
||||
|
||||
ExpectedErrorBehavior.SuccessEmpty =>
|
||||
output.Success && output.ExitCode == 0,
|
||||
|
||||
ExpectedErrorBehavior.SuccessOrError =>
|
||||
output.ExitCode == 0 || (!output.Success && !string.IsNullOrEmpty(output.Error)),
|
||||
|
||||
ExpectedErrorBehavior.SuccessOrSkip =>
|
||||
output.ExitCode == 0 || output.Error?.Contains("skip", StringComparison.OrdinalIgnoreCase) == true,
|
||||
|
||||
ExpectedErrorBehavior.TimeoutOrError =>
|
||||
!output.Success,
|
||||
|
||||
ExpectedErrorBehavior.Crash =>
|
||||
output.ExitCode < 0 || output.ExitCode > 128,
|
||||
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of error mode comparison.
|
||||
/// </summary>
|
||||
public sealed class ErrorModeComparisonResult
|
||||
{
|
||||
public bool Success { get; set; }
|
||||
public string? Error { get; set; }
|
||||
public int TestedScenarios { get; set; }
|
||||
public Dictionary<string, ToolErrorBehavior> ToolResults { get; set; } = new();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Error handling behavior for a single tool.
|
||||
/// </summary>
|
||||
public sealed class ToolErrorBehavior
|
||||
{
|
||||
public required string ToolName { get; init; }
|
||||
public List<ScenarioResult> ScenarioResults { get; set; } = [];
|
||||
public double PassRate { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result for a single error scenario.
|
||||
/// </summary>
|
||||
public sealed class ScenarioResult
|
||||
{
|
||||
public required string ScenarioName { get; init; }
|
||||
public ExpectedErrorBehavior ExpectedBehavior { get; set; }
|
||||
public int ActualExitCode { get; set; }
|
||||
public bool ActualSuccess { get; set; }
|
||||
public string? ErrorMessage { get; set; }
|
||||
public long DurationMs { get; set; }
|
||||
public bool BehaviorMatched { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Defines an error test scenario.
|
||||
/// </summary>
|
||||
public sealed class ErrorTestScenario
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
public required string ImageRef { get; init; }
|
||||
public required string Description { get; init; }
|
||||
public ExpectedErrorBehavior ExpectedBehavior { get; init; }
|
||||
public int? ExpectedExitCode { get; init; }
|
||||
public int TimeoutSeconds { get; init; } = 60;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expected error behavior categories.
|
||||
/// </summary>
|
||||
public enum ExpectedErrorBehavior
|
||||
{
|
||||
/// <summary>Tool should exit with non-zero and meaningful error message.</summary>
|
||||
GracefulError,
|
||||
|
||||
/// <summary>Tool should succeed but produce empty/minimal output.</summary>
|
||||
SuccessEmpty,
|
||||
|
||||
/// <summary>Either success or graceful error is acceptable.</summary>
|
||||
SuccessOrError,
|
||||
|
||||
/// <summary>Either success or skip message is acceptable.</summary>
|
||||
SuccessOrSkip,
|
||||
|
||||
/// <summary>Timeout or error is expected (network unreachable).</summary>
|
||||
TimeoutOrError,
|
||||
|
||||
/// <summary>Tool is expected to crash (for negative testing).</summary>
|
||||
Crash
|
||||
}
|
||||
Reference in New Issue
Block a user