Add determinism tests for verdict artifact generation and update SHA256 sums script
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
@@ -3,9 +3,10 @@
|
||||
// Task: T11 - Integration tests for crypto commands
|
||||
|
||||
using System.CommandLine;
|
||||
using System.CommandLine.IO;
|
||||
using System.CommandLine.Parsing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Spectre.Console;
|
||||
using Spectre.Console.Testing;
|
||||
using Xunit;
|
||||
using StellaOps.Cli.Commands;
|
||||
using StellaOps.Cryptography;
|
||||
@@ -119,11 +120,21 @@ public class CryptoCommandTests
|
||||
|
||||
// Act
|
||||
var console = new TestConsole();
|
||||
var exitCode = await command.InvokeAsync("sign --input /nonexistent/file.txt", console);
|
||||
var originalConsole = AnsiConsole.Console;
|
||||
int exitCode;
|
||||
try
|
||||
{
|
||||
AnsiConsole.Console = console;
|
||||
exitCode = await command.Parse("sign --input /nonexistent/file.txt").InvokeAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
AnsiConsole.Console = originalConsole;
|
||||
}
|
||||
|
||||
// Assert
|
||||
Assert.NotEqual(0, exitCode);
|
||||
var output = console.Error.ToString() ?? "";
|
||||
var output = console.Output.ToString();
|
||||
Assert.Contains("not found", output, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
@@ -143,11 +154,21 @@ public class CryptoCommandTests
|
||||
|
||||
// Act
|
||||
var console = new TestConsole();
|
||||
var exitCode = await command.InvokeAsync("profiles", console);
|
||||
var originalConsole = AnsiConsole.Console;
|
||||
int exitCode;
|
||||
try
|
||||
{
|
||||
AnsiConsole.Console = console;
|
||||
exitCode = await command.Parse("profiles").InvokeAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
AnsiConsole.Console = originalConsole;
|
||||
}
|
||||
|
||||
// Assert
|
||||
Assert.NotEqual(0, exitCode);
|
||||
var output = console.Out.ToString() ?? "";
|
||||
var output = console.Output.ToString();
|
||||
Assert.Contains("No crypto providers available", output, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
@@ -167,11 +188,21 @@ public class CryptoCommandTests
|
||||
|
||||
// Act
|
||||
var console = new TestConsole();
|
||||
var exitCode = await command.InvokeAsync("profiles", console);
|
||||
var originalConsole = AnsiConsole.Console;
|
||||
int exitCode;
|
||||
try
|
||||
{
|
||||
AnsiConsole.Console = console;
|
||||
exitCode = await command.Parse("profiles").InvokeAsync(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
AnsiConsole.Console = originalConsole;
|
||||
}
|
||||
|
||||
// Assert
|
||||
Assert.Equal(0, exitCode);
|
||||
var output = console.Out.ToString() ?? "";
|
||||
var output = console.Output.ToString();
|
||||
Assert.Contains("StubCryptoProvider", output);
|
||||
}
|
||||
|
||||
@@ -210,24 +241,18 @@ public class CryptoCommandTests
|
||||
{
|
||||
public string Name => "StubCryptoProvider";
|
||||
|
||||
public Task<byte[]> SignAsync(byte[] data, CryptoKeyReference keyRef, string algorithmId, CancellationToken ct = default)
|
||||
{
|
||||
return Task.FromResult(new byte[] { 0x01, 0x02, 0x03, 0x04 });
|
||||
}
|
||||
public bool Supports(CryptoCapability capability, string algorithmId) => true;
|
||||
|
||||
public Task<bool> VerifyAsync(byte[] data, byte[] signature, CryptoKeyReference keyRef, string algorithmId, CancellationToken ct = default)
|
||||
{
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
public IPasswordHasher GetPasswordHasher(string algorithmId) => throw new NotSupportedException();
|
||||
|
||||
public Task<byte[]> EncryptAsync(byte[] data, CryptoKeyReference keyRef, string algorithmId, CancellationToken ct = default)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
public ICryptoHasher GetHasher(string algorithmId) => throw new NotSupportedException();
|
||||
|
||||
public Task<byte[]> DecryptAsync(byte[] data, CryptoKeyReference keyRef, string algorithmId, CancellationToken ct = default)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
public ICryptoSigner GetSigner(string algorithmId, CryptoKeyReference keyReference) => throw new NotSupportedException();
|
||||
|
||||
public void UpsertSigningKey(CryptoSigningKey signingKey) => throw new NotSupportedException();
|
||||
|
||||
public bool RemoveSigningKey(string keyId) => false;
|
||||
|
||||
public IReadOnlyCollection<CryptoSigningKey> GetSigningKeys() => Array.Empty<CryptoSigningKey>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// CliExitCodeTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0010_cli_tests
|
||||
// Tasks: CLI-5100-001, CLI-5100-002, CLI-5100-003, CLI-5100-004
|
||||
// Description: Model CLI1 exit code tests for CLI tool
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using StellaOps.Cli.Commands;
|
||||
using StellaOps.Cli.Commands.Proof;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Cli.Tests.ExitCodes;
|
||||
|
||||
/// <summary>
|
||||
/// Exit code tests for the CLI tool.
|
||||
/// Implements Model CLI1 test requirements:
|
||||
/// - CLI-5100-001: Successful command → exit 0
|
||||
/// - CLI-5100-002: User error (bad arguments) → exit 1
|
||||
/// - CLI-5100-003: System error (API unavailable) → exit 2
|
||||
/// - CLI-5100-004: Permission denied → exit 3
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
[Trait("Category", "ExitCodes")]
|
||||
public sealed class CliExitCodeTests
|
||||
{
|
||||
// CLI-5100-001: Successful command exit codes
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_Success_IsZero()
|
||||
{
|
||||
// Assert
|
||||
ProofExitCodes.Success.Should().Be(0, "successful command should return exit 0");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineExitCodes_Success_IsZero()
|
||||
{
|
||||
// Assert
|
||||
OfflineExitCodes.Success.Should().Be(0, "successful command should return exit 0");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_Success_IsZero()
|
||||
{
|
||||
// Assert
|
||||
DriftExitCodes.Success.Should().Be(0, "successful command should return exit 0");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_IsSuccess_ReturnsTrueForSuccessCodes()
|
||||
{
|
||||
// Assert - success range is 0-9
|
||||
DriftExitCodes.IsSuccess(0).Should().BeTrue();
|
||||
DriftExitCodes.IsSuccess(1).Should().BeTrue();
|
||||
DriftExitCodes.IsSuccess(2).Should().BeTrue();
|
||||
DriftExitCodes.IsSuccess(9).Should().BeTrue();
|
||||
DriftExitCodes.IsSuccess(10).Should().BeFalse();
|
||||
}
|
||||
|
||||
// CLI-5100-002: User error exit codes (bad arguments)
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_InputError_IsNonZero()
|
||||
{
|
||||
// Assert
|
||||
ProofExitCodes.InputError.Should().BeGreaterThan(0, "user error should return non-zero exit code");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_PolicyViolation_IsOne()
|
||||
{
|
||||
// Assert - policy violation is a user-facing error (not system)
|
||||
ProofExitCodes.PolicyViolation.Should().Be(1, "policy violation should return exit 1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineExitCodes_FileNotFound_IsOne()
|
||||
{
|
||||
// Assert - missing file is user error
|
||||
OfflineExitCodes.FileNotFound.Should().Be(1, "file not found should return exit 1");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_InputError_IsTen()
|
||||
{
|
||||
// Assert - input error in drift range
|
||||
DriftExitCodes.InputError.Should().Be(10, "input error should be in error range");
|
||||
DriftExitCodes.IsError(DriftExitCodes.InputError).Should().BeTrue();
|
||||
}
|
||||
|
||||
// CLI-5100-003: System error exit codes (API unavailable)
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_SystemError_IsTwo()
|
||||
{
|
||||
// Assert
|
||||
ProofExitCodes.SystemError.Should().Be(2, "system error should return exit 2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_NetworkError_IsNonZero()
|
||||
{
|
||||
// Assert
|
||||
DriftExitCodes.NetworkError.Should().Be(14, "network error should be non-zero");
|
||||
DriftExitCodes.IsError(DriftExitCodes.NetworkError).Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_StorageError_IsNonZero()
|
||||
{
|
||||
// Assert
|
||||
DriftExitCodes.StorageError.Should().Be(12, "storage error should be non-zero");
|
||||
DriftExitCodes.IsError(DriftExitCodes.StorageError).Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_UnknownError_Is99()
|
||||
{
|
||||
// Assert
|
||||
DriftExitCodes.UnknownError.Should().Be(99, "unknown error should be 99");
|
||||
}
|
||||
|
||||
// CLI-5100-004: Permission/verification denied exit codes
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_VerificationFailed_IsThree()
|
||||
{
|
||||
// Assert
|
||||
ProofExitCodes.VerificationFailed.Should().Be(3, "verification failed should return exit 3");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_TrustAnchorError_IsFour()
|
||||
{
|
||||
// Assert
|
||||
ProofExitCodes.TrustAnchorError.Should().Be(4, "trust anchor error should return exit 4");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_KeyRevoked_IsSix()
|
||||
{
|
||||
// Assert
|
||||
ProofExitCodes.KeyRevoked.Should().Be(6, "key revoked should return exit 6");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineExitCodes_SignatureFailure_IsThree()
|
||||
{
|
||||
// Assert
|
||||
OfflineExitCodes.SignatureFailure.Should().Be(3, "signature failure should return exit 3");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineExitCodes_DsseVerificationFailed_IsFive()
|
||||
{
|
||||
// Assert
|
||||
OfflineExitCodes.DsseVerificationFailed.Should().Be(5, "DSSE verification failure should return exit 5");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineExitCodes_PolicyDenied_IsNine()
|
||||
{
|
||||
// Assert
|
||||
OfflineExitCodes.PolicyDenied.Should().Be(9, "policy denied should return exit 9");
|
||||
}
|
||||
|
||||
// Exit code naming and description tests
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_GetName_ReturnsCorrectNames()
|
||||
{
|
||||
// Assert
|
||||
DriftExitCodes.GetName(DriftExitCodes.Success).Should().Be("SUCCESS");
|
||||
DriftExitCodes.GetName(DriftExitCodes.InputError).Should().Be("INPUT_ERROR");
|
||||
DriftExitCodes.GetName(DriftExitCodes.NetworkError).Should().Be("NETWORK_ERROR");
|
||||
DriftExitCodes.GetName(999).Should().Be("UNKNOWN_ERROR");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_GetDescription_ReturnsNonEmptyDescriptions()
|
||||
{
|
||||
// Assert - all exit codes should have descriptions
|
||||
DriftExitCodes.GetDescription(DriftExitCodes.Success).Should().NotBeNullOrEmpty();
|
||||
DriftExitCodes.GetDescription(DriftExitCodes.InputError).Should().NotBeNullOrEmpty();
|
||||
DriftExitCodes.GetDescription(DriftExitCodes.NetworkError).Should().NotBeNullOrEmpty();
|
||||
DriftExitCodes.GetDescription(999).Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_GetDescription_ReturnsNonEmptyDescriptions()
|
||||
{
|
||||
// Assert - all exit codes should have descriptions
|
||||
ProofExitCodes.GetDescription(ProofExitCodes.Success).Should().NotBeNullOrEmpty();
|
||||
ProofExitCodes.GetDescription(ProofExitCodes.PolicyViolation).Should().NotBeNullOrEmpty();
|
||||
ProofExitCodes.GetDescription(ProofExitCodes.SystemError).Should().NotBeNullOrEmpty();
|
||||
ProofExitCodes.GetDescription(999).Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
// Exit code range consistency tests
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_SuccessRange_IsZeroToNine()
|
||||
{
|
||||
// Assert - verify success range boundaries
|
||||
DriftExitCodes.IsSuccess(0).Should().BeTrue("0 should be success");
|
||||
DriftExitCodes.IsSuccess(9).Should().BeTrue("9 should be success");
|
||||
DriftExitCodes.IsSuccess(10).Should().BeFalse("10 should not be success");
|
||||
DriftExitCodes.IsSuccess(-1).Should().BeFalse("negative should not be success");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_ErrorRange_IsTenOrHigher()
|
||||
{
|
||||
// Assert - verify error range boundaries
|
||||
DriftExitCodes.IsError(9).Should().BeFalse("9 should not be error");
|
||||
DriftExitCodes.IsError(10).Should().BeTrue("10 should be error");
|
||||
DriftExitCodes.IsError(99).Should().BeTrue("99 should be error");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftExitCodes_BlockingCodes_AreCorrect()
|
||||
{
|
||||
// Assert - blocking codes should stop CI/CD
|
||||
DriftExitCodes.IsBlocking(DriftExitCodes.KevReachable).Should().BeTrue();
|
||||
DriftExitCodes.IsBlocking(DriftExitCodes.AffectedReachable).Should().BeTrue();
|
||||
DriftExitCodes.IsBlocking(DriftExitCodes.PolicyBlocked).Should().BeTrue();
|
||||
|
||||
// Non-blocking codes
|
||||
DriftExitCodes.IsBlocking(DriftExitCodes.Success).Should().BeFalse();
|
||||
DriftExitCodes.IsBlocking(DriftExitCodes.SuccessWithInfoDrift).Should().BeFalse();
|
||||
}
|
||||
|
||||
// POSIX convention tests
|
||||
|
||||
[Fact]
|
||||
public void AllExitCodes_FollowPosixConvention_ZeroIsSuccess()
|
||||
{
|
||||
// Assert - verify POSIX convention across all exit code classes
|
||||
ProofExitCodes.Success.Should().Be(0);
|
||||
OfflineExitCodes.Success.Should().Be(0);
|
||||
DriftExitCodes.Success.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineExitCodes_Cancelled_Is130()
|
||||
{
|
||||
// Assert - SIGINT standard exit code
|
||||
OfflineExitCodes.Cancelled.Should().Be(130, "SIGINT cancellation should be 128+2=130");
|
||||
}
|
||||
|
||||
// Exit code uniqueness tests
|
||||
|
||||
[Fact]
|
||||
public void ProofExitCodes_AllCodesAreDistinct()
|
||||
{
|
||||
// Arrange
|
||||
var codes = new[]
|
||||
{
|
||||
ProofExitCodes.Success,
|
||||
ProofExitCodes.PolicyViolation,
|
||||
ProofExitCodes.SystemError,
|
||||
ProofExitCodes.VerificationFailed,
|
||||
ProofExitCodes.TrustAnchorError,
|
||||
ProofExitCodes.RekorVerificationFailed,
|
||||
ProofExitCodes.KeyRevoked,
|
||||
ProofExitCodes.OfflineModeError,
|
||||
ProofExitCodes.InputError
|
||||
};
|
||||
|
||||
// Assert
|
||||
codes.Should().OnlyHaveUniqueItems("all exit codes should be distinct");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineExitCodes_AllCodesAreDistinct()
|
||||
{
|
||||
// Arrange
|
||||
var codes = new[]
|
||||
{
|
||||
OfflineExitCodes.Success,
|
||||
OfflineExitCodes.FileNotFound,
|
||||
OfflineExitCodes.ChecksumMismatch,
|
||||
OfflineExitCodes.SignatureFailure,
|
||||
OfflineExitCodes.FormatError,
|
||||
OfflineExitCodes.DsseVerificationFailed,
|
||||
OfflineExitCodes.RekorVerificationFailed,
|
||||
OfflineExitCodes.ImportFailed,
|
||||
OfflineExitCodes.VersionNonMonotonic,
|
||||
OfflineExitCodes.PolicyDenied,
|
||||
OfflineExitCodes.SelftestFailed,
|
||||
OfflineExitCodes.ValidationFailed,
|
||||
OfflineExitCodes.VerificationFailed,
|
||||
OfflineExitCodes.PolicyLoadFailed,
|
||||
OfflineExitCodes.Cancelled
|
||||
};
|
||||
|
||||
// Assert
|
||||
codes.Should().OnlyHaveUniqueItems("all exit codes should be distinct");
|
||||
}
|
||||
|
||||
// DriftCommandResult tests
|
||||
|
||||
[Fact]
|
||||
public void DriftCommandResult_CanBeCreated_WithRequiredProperties()
|
||||
{
|
||||
// Arrange & Act
|
||||
var result = new DriftCommandResult
|
||||
{
|
||||
ExitCode = DriftExitCodes.Success,
|
||||
Message = "No drift detected"
|
||||
};
|
||||
|
||||
// Assert
|
||||
result.ExitCode.Should().Be(0);
|
||||
result.Message.Should().Be("No drift detected");
|
||||
result.DeltaReachable.Should().Be(0);
|
||||
result.DeltaUnreachable.Should().Be(0);
|
||||
result.HasKevReachable.Should().BeFalse();
|
||||
result.BlockedBy.Should().BeNull();
|
||||
result.Suggestion.Should().BeNull();
|
||||
result.SarifOutputPath.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DriftCommandResult_CanBeCreated_WithAllProperties()
|
||||
{
|
||||
// Arrange & Act
|
||||
var result = new DriftCommandResult
|
||||
{
|
||||
ExitCode = DriftExitCodes.KevReachable,
|
||||
Message = "KEV now reachable",
|
||||
DeltaReachable = 5,
|
||||
DeltaUnreachable = 2,
|
||||
HasKevReachable = true,
|
||||
BlockedBy = "kev-policy",
|
||||
Suggestion = "Upgrade vulnerable package",
|
||||
SarifOutputPath = "/tmp/results.sarif"
|
||||
};
|
||||
|
||||
// Assert
|
||||
result.ExitCode.Should().Be(DriftExitCodes.KevReachable);
|
||||
result.Message.Should().Be("KEV now reachable");
|
||||
result.DeltaReachable.Should().Be(5);
|
||||
result.DeltaUnreachable.Should().Be(2);
|
||||
result.HasKevReachable.Should().BeTrue();
|
||||
result.BlockedBy.Should().Be("kev-policy");
|
||||
result.Suggestion.Should().Be("Upgrade vulnerable package");
|
||||
result.SarifOutputPath.Should().Be("/tmp/results.sarif");
|
||||
}
|
||||
}
|
||||
@@ -23,8 +23,12 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
|
||||
<PackageReference Include="Moq" Version="4.20.72" />
|
||||
<PackageReference Include="Spectre.Console.Testing" Version="0.48.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
||||
<ProjectReference Include="../../StellaOps.Cli/StellaOps.Cli.csproj" />
|
||||
<ProjectReference Include="../../__Libraries/StellaOps.Cli.Plugins.NonCore/StellaOps.Cli.Plugins.NonCore.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Configuration/StellaOps.Configuration.csproj" />
|
||||
|
||||
Reference in New Issue
Block a user