Add tests for SBOM generation determinism across multiple formats
- Created `StellaOps.TestKit.Tests` project for unit tests related to determinism. - Implemented `DeterminismManifestTests` to validate deterministic output for canonical bytes and strings, file read/write operations, and error handling for invalid schema versions. - Added `SbomDeterminismTests` to ensure identical inputs produce consistent SBOMs across SPDX 3.0.1 and CycloneDX 1.6/1.7 formats, including parallel execution tests. - Updated project references in `StellaOps.Integration.Determinism` to include the new determinism testing library.
This commit is contained in:
master
StellaOps Bot
@@ -0,0 +1,229 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ScanResultIdempotencyTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0001_scanner_tests
|
||||
// Task: SCANNER-5100-014
|
||||
// Description: Model S1 idempotency tests for Scanner scan results storage
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Infrastructure.Postgres.Options;
|
||||
using StellaOps.Scanner.Storage;
|
||||
using StellaOps.Scanner.Storage.Entities;
|
||||
using StellaOps.Scanner.Storage.Postgres;
|
||||
using StellaOps.TestKit;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Storage.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Idempotency tests for scan result storage operations.
|
||||
/// Implements Model S1 (Storage/Postgres) test requirements:
|
||||
/// - Insert same entity twice → no duplicates
|
||||
/// - Same manifest hash → same record returned
|
||||
/// - Update operations are idempotent
|
||||
/// </summary>
|
||||
[Collection("scanner-postgres")]
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
[Trait("Category", TestCategories.StorageIdempotency)]
|
||||
public sealed class ScanResultIdempotencyTests : IAsyncLifetime
|
||||
{
|
||||
private readonly ScannerPostgresFixture _fixture;
|
||||
private PostgresScanManifestRepository _manifestRepository = null!;
|
||||
private ScannerDataSource _dataSource = null!;
|
||||
|
||||
public ScanResultIdempotencyTests(ScannerPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.TruncateAllTablesAsync();
|
||||
|
||||
var options = new ScannerStorageOptions
|
||||
{
|
||||
Postgres = new PostgresOptions
|
||||
{
|
||||
ConnectionString = _fixture.ConnectionString,
|
||||
SchemaName = _fixture.SchemaName
|
||||
}
|
||||
};
|
||||
|
||||
_dataSource = new ScannerDataSource(Options.Create(options), NullLogger<ScannerDataSource>.Instance);
|
||||
_manifestRepository = new PostgresScanManifestRepository(_dataSource);
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task SaveAsync_SameManifestHash_Twice_CanRetrieveByHash()
|
||||
{
|
||||
// Arrange
|
||||
var manifest1 = CreateManifest("sha256:manifest1");
|
||||
var manifest2 = CreateManifest("sha256:manifest1"); // Same hash
|
||||
|
||||
// Act
|
||||
var saved1 = await _manifestRepository.SaveAsync(manifest1);
|
||||
|
||||
// Try to save second with same hash - depending on DB constraint
|
||||
// this might fail or create a new record
|
||||
try
|
||||
{
|
||||
var saved2 = await _manifestRepository.SaveAsync(manifest2);
|
||||
|
||||
// If it succeeds, verify we can get by hash
|
||||
var retrieved = await _manifestRepository.GetByHashAsync("sha256:manifest1");
|
||||
retrieved.Should().NotBeNull();
|
||||
}
|
||||
catch (Npgsql.PostgresException)
|
||||
{
|
||||
// Expected if manifest_hash has unique constraint
|
||||
// Verify the first one still exists
|
||||
var retrieved = await _manifestRepository.GetByHashAsync("sha256:manifest1");
|
||||
retrieved.Should().NotBeNull();
|
||||
retrieved!.ManifestId.Should().Be(saved1.ManifestId);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByHashAsync_SameHash_ReturnsConsistentResult()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateManifest("sha256:consistent");
|
||||
await _manifestRepository.SaveAsync(manifest);
|
||||
|
||||
// Act - Query same hash multiple times
|
||||
var results = new List<ScanManifestRow?>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(await _manifestRepository.GetByHashAsync("sha256:consistent"));
|
||||
}
|
||||
|
||||
// Assert - All should return same record
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Should().NotBeNull();
|
||||
r!.ManifestHash.Should().Be("sha256:consistent");
|
||||
});
|
||||
|
||||
var distinctIds = results.Where(r => r != null).Select(r => r!.ManifestId).Distinct().ToList();
|
||||
distinctIds.Should().HaveCount(1, "same hash should always return same manifest");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByScanIdAsync_SameId_ReturnsConsistentResult()
|
||||
{
|
||||
// Arrange
|
||||
var scanId = Guid.NewGuid();
|
||||
var manifest = CreateManifest("sha256:byscan", scanId);
|
||||
await _manifestRepository.SaveAsync(manifest);
|
||||
|
||||
// Act - Query same scan ID multiple times
|
||||
var results = new List<ScanManifestRow?>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(await _manifestRepository.GetByScanIdAsync(scanId));
|
||||
}
|
||||
|
||||
// Assert - All should return same record
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Should().NotBeNull();
|
||||
r!.ScanId.Should().Be(scanId);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkCompletedAsync_Twice_IsIdempotent()
|
||||
{
|
||||
// Arrange
|
||||
var manifest = CreateManifest("sha256:complete");
|
||||
var saved = await _manifestRepository.SaveAsync(manifest);
|
||||
|
||||
var completedAt1 = DateTimeOffset.UtcNow;
|
||||
var completedAt2 = DateTimeOffset.UtcNow.AddMinutes(1);
|
||||
|
||||
// Act - Mark completed twice
|
||||
await _manifestRepository.MarkCompletedAsync(saved.ManifestId, completedAt1);
|
||||
var after1 = await _manifestRepository.GetByHashAsync("sha256:complete");
|
||||
|
||||
await _manifestRepository.MarkCompletedAsync(saved.ManifestId, completedAt2);
|
||||
var after2 = await _manifestRepository.GetByHashAsync("sha256:complete");
|
||||
|
||||
// Assert - Both should succeed, second updates the timestamp
|
||||
after1.Should().NotBeNull();
|
||||
after1!.ScanCompletedAt.Should().NotBeNull();
|
||||
|
||||
after2.Should().NotBeNull();
|
||||
after2!.ScanCompletedAt.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MarkCompletedAsync_NonExistent_DoesNotThrow()
|
||||
{
|
||||
// Arrange
|
||||
var nonExistentId = Guid.NewGuid();
|
||||
|
||||
// Act
|
||||
var action = async () =>
|
||||
await _manifestRepository.MarkCompletedAsync(nonExistentId, DateTimeOffset.UtcNow);
|
||||
|
||||
// Assert - Should not throw (0 rows affected is OK)
|
||||
await action.Should().NotThrowAsync();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SaveAsync_MultipleDifferentScans_AllPersisted()
|
||||
{
|
||||
// Arrange
|
||||
var manifests = Enumerable.Range(0, 5)
|
||||
.Select(i => CreateManifest($"sha256:multi{i}"))
|
||||
.ToList();
|
||||
|
||||
// Act
|
||||
foreach (var manifest in manifests)
|
||||
{
|
||||
await _manifestRepository.SaveAsync(manifest);
|
||||
}
|
||||
|
||||
// Assert - All should be retrievable
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
var retrieved = await _manifestRepository.GetByHashAsync($"sha256:multi{i}");
|
||||
retrieved.Should().NotBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SaveAsync_MultipleManifstsForSameScan_AllRetrievable()
|
||||
{
|
||||
// Arrange - Same scan ID, different manifests (e.g., scan retry)
|
||||
var scanId = Guid.NewGuid();
|
||||
var manifest1 = CreateManifest("sha256:retry1", scanId);
|
||||
var manifest2 = CreateManifest("sha256:retry2", scanId);
|
||||
|
||||
// Act
|
||||
await _manifestRepository.SaveAsync(manifest1);
|
||||
await _manifestRepository.SaveAsync(manifest2);
|
||||
|
||||
// Assert - GetByScanId returns most recent
|
||||
var retrieved = await _manifestRepository.GetByScanIdAsync(scanId);
|
||||
retrieved.Should().NotBeNull();
|
||||
// Should return one of them (most recent by created_at)
|
||||
}
|
||||
|
||||
private static ScanManifestRow CreateManifest(string hash, Guid? scanId = null) => new()
|
||||
{
|
||||
ScanId = scanId ?? Guid.NewGuid(),
|
||||
ManifestHash = hash,
|
||||
SbomHash = "sha256:sbom" + Guid.NewGuid().ToString("N")[..8],
|
||||
RulesHash = "sha256:rules" + Guid.NewGuid().ToString("N")[..8],
|
||||
FeedHash = "sha256:feed" + Guid.NewGuid().ToString("N")[..8],
|
||||
PolicyHash = "sha256:policy" + Guid.NewGuid().ToString("N")[..8],
|
||||
ScanStartedAt = DateTimeOffset.UtcNow,
|
||||
ManifestContent = """{"version": "1.0", "scanner": "stellaops"}""",
|
||||
ScannerVersion = "1.0.0"
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user