Add tests for SBOM generation determinism across multiple formats
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
StellaOps Bot
@@ -7,7 +7,6 @@ using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Json;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
@@ -22,19 +21,25 @@ using StellaOps.Cli.Services.Models;
|
||||
using StellaOps.Cli.Services.Models.Transport;
|
||||
using StellaOps.Cli.Tests.Testing;
|
||||
using StellaOps.Scanner.EntryTrace;
|
||||
using StellaOps.Cryptography;
|
||||
using System.Linq;
|
||||
|
||||
namespace StellaOps.Cli.Tests.Services;
|
||||
|
||||
public sealed class BackendOperationsClientTests
|
||||
{
|
||||
private static readonly ICryptoHash CryptoHash = DefaultCryptoHash.CreateForTests();
|
||||
|
||||
private static string ComputeSha256Hex(ReadOnlySpan<byte> data)
|
||||
=> CryptoHash.ComputeHashHex(data, HashAlgorithms.Sha256);
|
||||
|
||||
[Fact]
|
||||
public async Task DownloadScannerAsync_VerifiesDigestAndWritesMetadata()
|
||||
{
|
||||
using var temp = new TempDirectory();
|
||||
|
||||
var contentBytes = Encoding.UTF8.GetBytes("scanner-blob");
|
||||
var digestHex = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant();
|
||||
var digestHex = ComputeSha256Hex(contentBytes);
|
||||
|
||||
var handler = new StubHttpMessageHandler((request, _) =>
|
||||
{
|
||||
@@ -63,7 +68,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var targetPath = Path.Combine(temp.Path, "scanner.tar.gz");
|
||||
var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: true, CancellationToken.None);
|
||||
@@ -85,6 +90,7 @@ public sealed class BackendOperationsClientTests
|
||||
using var temp = new TempDirectory();
|
||||
|
||||
var contentBytes = Encoding.UTF8.GetBytes("scanner-data");
|
||||
var wrongDigestHex = ComputeSha256Hex(Encoding.UTF8.GetBytes("wrong-data"));
|
||||
var handler = new StubHttpMessageHandler((request, _) =>
|
||||
{
|
||||
var response = new HttpResponseMessage(HttpStatusCode.OK)
|
||||
@@ -92,7 +98,7 @@ public sealed class BackendOperationsClientTests
|
||||
Content = new ByteArrayContent(contentBytes),
|
||||
RequestMessage = request
|
||||
};
|
||||
response.Headers.Add("X-StellaOps-Digest", "sha256:deadbeef");
|
||||
response.Headers.Add("X-StellaOps-Digest", $"sha256:{wrongDigestHex}");
|
||||
return response;
|
||||
});
|
||||
|
||||
@@ -109,7 +115,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var targetPath = Path.Combine(temp.Path, "scanner.tar.gz");
|
||||
|
||||
@@ -123,7 +129,7 @@ public sealed class BackendOperationsClientTests
|
||||
using var temp = new TempDirectory();
|
||||
|
||||
var successBytes = Encoding.UTF8.GetBytes("success");
|
||||
var digestHex = Convert.ToHexString(SHA256.HashData(successBytes)).ToLowerInvariant();
|
||||
var digestHex = ComputeSha256Hex(successBytes);
|
||||
var attempts = 0;
|
||||
|
||||
var handler = new StubHttpMessageHandler(
|
||||
@@ -161,7 +167,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var targetPath = Path.Combine(temp.Path, "scanner.tar.gz");
|
||||
var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: false, CancellationToken.None);
|
||||
@@ -212,7 +218,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
await client.UploadScanResultsAsync(filePath, CancellationToken.None);
|
||||
|
||||
@@ -250,7 +256,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() => client.UploadScanResultsAsync(filePath, CancellationToken.None));
|
||||
Assert.Equal(2, attempts);
|
||||
@@ -316,7 +322,7 @@ public sealed class BackendOperationsClientTests
|
||||
BackendUrl = "https://scanner.example"
|
||||
};
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var result = await client.GetEntryTraceAsync(scanId, CancellationToken.None);
|
||||
|
||||
@@ -345,7 +351,7 @@ public sealed class BackendOperationsClientTests
|
||||
BackendUrl = "https://scanner.example"
|
||||
};
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var result = await client.GetEntryTraceAsync("scan-missing", CancellationToken.None);
|
||||
Assert.Null(result);
|
||||
@@ -379,7 +385,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var options = new StellaOpsCliOptions { BackendUrl = "https://concelier.example" };
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var result = await client.TriggerJobAsync("export:json", new Dictionary<string, object?>(), CancellationToken.None);
|
||||
|
||||
@@ -414,7 +420,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var options = new StellaOpsCliOptions { BackendUrl = "https://concelier.example" };
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var result = await client.TriggerJobAsync("export:json", new Dictionary<string, object?>(), CancellationToken.None);
|
||||
|
||||
@@ -467,7 +473,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var tokenClient = new StubTokenClient();
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), tokenClient);
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash, tokenClient);
|
||||
|
||||
var result = await client.TriggerJobAsync("test", new Dictionary<string, object?>(), CancellationToken.None);
|
||||
|
||||
@@ -517,7 +523,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var tokenClient = new StubTokenClient();
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), tokenClient);
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash, tokenClient);
|
||||
|
||||
var exception = await Assert.ThrowsAsync<InvalidOperationException>(() => client.TriggerJobAsync("test", new Dictionary<string, object?>(), CancellationToken.None));
|
||||
Assert.Contains("Authority.BackfillReason", exception.Message, StringComparison.Ordinal);
|
||||
@@ -570,7 +576,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var tokenClient = new StubTokenClient();
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), tokenClient);
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash, tokenClient);
|
||||
|
||||
var result = await client.TriggerJobAsync("test", new Dictionary<string, object?>(), CancellationToken.None);
|
||||
|
||||
@@ -643,7 +649,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var labels = new ReadOnlyDictionary<string, string>(new Dictionary<string, string> { ["app"] = "payments" });
|
||||
var imagesList = new ReadOnlyCollection<string>(new List<string>
|
||||
@@ -693,8 +699,8 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var bundleBytes = Encoding.UTF8.GetBytes("bundle-data");
|
||||
var manifestBytes = Encoding.UTF8.GetBytes("{\"artifacts\":[]}");
|
||||
var bundleDigest = Convert.ToHexString(SHA256.HashData(bundleBytes)).ToLowerInvariant();
|
||||
var manifestDigest = Convert.ToHexString(SHA256.HashData(manifestBytes)).ToLowerInvariant();
|
||||
var bundleDigest = ComputeSha256Hex(bundleBytes);
|
||||
var manifestDigest = ComputeSha256Hex(manifestBytes);
|
||||
|
||||
var metadataPayload = JsonSerializer.Serialize(new
|
||||
{
|
||||
@@ -762,7 +768,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var result = await client.DownloadOfflineKitAsync(null, temp.Path, overwrite: false, resume: false, CancellationToken.None);
|
||||
|
||||
@@ -785,8 +791,8 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var bundleBytes = Encoding.UTF8.GetBytes("partial-download-data");
|
||||
var manifestBytes = Encoding.UTF8.GetBytes("{\"manifest\":true}");
|
||||
var bundleDigest = Convert.ToHexString(SHA256.HashData(bundleBytes)).ToLowerInvariant();
|
||||
var manifestDigest = Convert.ToHexString(SHA256.HashData(manifestBytes)).ToLowerInvariant();
|
||||
var bundleDigest = ComputeSha256Hex(bundleBytes);
|
||||
var manifestDigest = ComputeSha256Hex(manifestBytes);
|
||||
|
||||
var metadataJson = JsonSerializer.Serialize(new
|
||||
{
|
||||
@@ -842,7 +848,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var result = await client.DownloadOfflineKitAsync(null, temp.Path, overwrite: false, resume: true, CancellationToken.None);
|
||||
|
||||
@@ -862,8 +868,8 @@ public sealed class BackendOperationsClientTests
|
||||
await File.WriteAllBytesAsync(bundlePath, bundleBytes);
|
||||
await File.WriteAllBytesAsync(manifestPath, manifestBytes);
|
||||
|
||||
var bundleDigest = Convert.ToHexString(SHA256.HashData(bundleBytes)).ToLowerInvariant();
|
||||
var manifestDigest = Convert.ToHexString(SHA256.HashData(manifestBytes)).ToLowerInvariant();
|
||||
var bundleDigest = ComputeSha256Hex(bundleBytes);
|
||||
var manifestDigest = ComputeSha256Hex(manifestBytes);
|
||||
|
||||
var metadata = new OfflineKitMetadataDocument
|
||||
{
|
||||
@@ -898,7 +904,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var request = new OfflineKitImportRequest(
|
||||
bundlePath,
|
||||
@@ -982,7 +988,7 @@ public sealed class BackendOperationsClientTests
|
||||
};
|
||||
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var status = await client.GetOfflineKitStatusAsync(CancellationToken.None);
|
||||
|
||||
@@ -1126,7 +1132,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var options = new StellaOpsCliOptions { BackendUrl = "https://policy.example" };
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var sbomSet = new ReadOnlyCollection<string>(new List<string> { "sbom:A", "sbom:B" });
|
||||
var environment = new ReadOnlyDictionary<string, object?>(new Dictionary<string, object?>(StringComparer.Ordinal)
|
||||
@@ -1193,7 +1199,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var options = new StellaOpsCliOptions { BackendUrl = "https://policy.example" };
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var input = new PolicySimulationInput(
|
||||
null,
|
||||
@@ -1257,7 +1263,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var options = new StellaOpsCliOptions { BackendUrl = "https://policy.example" };
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var request = new PolicyActivationRequest(
|
||||
RunNow: true,
|
||||
@@ -1321,7 +1327,7 @@ public sealed class BackendOperationsClientTests
|
||||
|
||||
var options = new StellaOpsCliOptions { BackendUrl = "https://policy.example" };
|
||||
var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug));
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>());
|
||||
var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger<BackendOperationsClient>(), CryptoHash);
|
||||
|
||||
var request = new PolicyActivationRequest(false, null, null, false, null, null);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user