Add determinism tests for verdict artifact generation and update SHA256 sums script

- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering.
- Created helper methods for generating sample verdict inputs and computing canonical hashes.
- Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics.
- Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
2025-12-24 02:17:34 +02:00
parent e59921374e
commit 7503c19b8f
390 changed files with 37389 additions and 5380 deletions

View File

@@ -0,0 +1,651 @@
// -----------------------------------------------------------------------------
// EvidenceBundleImmutabilityTests.cs
// Sprint: SPRINT_5100_0010_0001_evidencelocker_tests
// Tasks: EVIDENCE-5100-001, EVIDENCE-5100-002, EVIDENCE-5100-003
// Description: Model L0+S1 immutability tests for EvidenceLocker bundles
// -----------------------------------------------------------------------------
using System;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Docker.DotNet;
using DotNet.Testcontainers.Builders;
using DotNet.Testcontainers.Configurations;
using DotNet.Testcontainers.Containers;
using Microsoft.Extensions.Logging.Abstractions;
using Npgsql;
using StellaOps.EvidenceLocker.Core.Configuration;
using StellaOps.EvidenceLocker.Core.Domain;
using StellaOps.EvidenceLocker.Core.Repositories;
using StellaOps.EvidenceLocker.Infrastructure.Db;
using StellaOps.EvidenceLocker.Infrastructure.Repositories;
using Xunit;
namespace StellaOps.EvidenceLocker.Tests;
/// <summary>
/// Immutability tests for EvidenceLocker bundles.
/// Implements Model L0+S1 test requirements:
/// - Once stored, artifact cannot be overwritten (reject or version)
/// - Simultaneous writes to same key → deterministic behavior (first wins or explicit error)
/// - Same key + different payload → new version created (if versioning enabled)
/// </summary>
[Trait("Category", "Integration")]
[Trait("Category", "Immutability")]
public sealed class EvidenceBundleImmutabilityTests : IAsyncLifetime
{
private readonly PostgreSqlTestcontainer _postgres;
private EvidenceLockerDataSource? _dataSource;
private IEvidenceLockerMigrationRunner? _migrationRunner;
private IEvidenceBundleRepository? _repository;
private string? _skipReason;
public EvidenceBundleImmutabilityTests()
{
_postgres = new TestcontainersBuilder<PostgreSqlTestcontainer>()
.WithDatabase(new PostgreSqlTestcontainerConfiguration
{
Database = "evidence_locker_immutability_tests",
Username = "postgres",
Password = "postgres"
})
.WithCleanUp(true)
.Build();
}
// EVIDENCE-5100-001: Once stored, artifact cannot be overwritten
[Fact]
public async Task CreateBundle_SameId_SecondInsertFails()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle1 = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now,
Description: "First bundle");
var bundle2 = new EvidenceBundle(
bundleId, // Same ID
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('b', 64), // Different hash
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource2",
CreatedAt: now,
UpdatedAt: now,
Description: "Second bundle with same ID");
// First insert should succeed
await _repository!.CreateBundleAsync(bundle1, cancellationToken);
// Second insert with same ID should fail
await Assert.ThrowsAsync<PostgresException>(async () =>
await _repository.CreateBundleAsync(bundle2, cancellationToken));
}
[Fact]
public async Task CreateBundle_SameIdDifferentTenant_BothSucceed()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenant1 = TenantId.FromGuid(Guid.NewGuid());
var tenant2 = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle1 = new EvidenceBundle(
bundleId,
tenant1,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenant1}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
var bundle2 = new EvidenceBundle(
bundleId, // Same bundle ID
tenant2, // Different tenant
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('b', 64),
StorageKey: $"tenants/{tenant2}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
// Both should succeed - different tenants can have same bundle ID
await _repository!.CreateBundleAsync(bundle1, cancellationToken);
await _repository.CreateBundleAsync(bundle2, cancellationToken);
// Verify both exist
var exists1 = await _repository.ExistsAsync(bundleId, tenant1, cancellationToken);
var exists2 = await _repository.ExistsAsync(bundleId, tenant2, cancellationToken);
Assert.True(exists1, "Bundle should exist for tenant1");
Assert.True(exists2, "Bundle should exist for tenant2");
}
[Fact]
public async Task SealedBundle_CannotBeModified()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
await _repository!.CreateBundleAsync(bundle, cancellationToken);
// Seal the bundle
await _repository.MarkBundleSealedAsync(
bundleId,
tenantId,
EvidenceBundleStatus.Sealed,
now.AddMinutes(1),
cancellationToken);
// Verify bundle is sealed
var fetched = await _repository.GetBundleAsync(bundleId, tenantId, cancellationToken);
Assert.NotNull(fetched);
Assert.Equal(EvidenceBundleStatus.Sealed, fetched.Bundle.Status);
Assert.NotNull(fetched.Bundle.SealedAt);
}
[Fact]
public async Task Bundle_ExistsCheck_ReturnsCorrectState()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var nonExistentBundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
// Before creation
var existsBefore = await _repository!.ExistsAsync(bundleId, tenantId, cancellationToken);
Assert.False(existsBefore, "Bundle should not exist before creation");
// Create bundle
var bundle = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
await _repository.CreateBundleAsync(bundle, cancellationToken);
// After creation
var existsAfter = await _repository.ExistsAsync(bundleId, tenantId, cancellationToken);
Assert.True(existsAfter, "Bundle should exist after creation");
// Non-existent bundle
var existsNonExistent = await _repository.ExistsAsync(nonExistentBundleId, tenantId, cancellationToken);
Assert.False(existsNonExistent, "Non-existent bundle should not exist");
}
// EVIDENCE-5100-002: Simultaneous writes to same key → deterministic behavior
[Fact]
public async Task ConcurrentCreates_SameId_ExactlyOneFails()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle1 = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource1",
CreatedAt: now,
UpdatedAt: now,
Description: "Concurrent bundle 1");
var bundle2 = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('b', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource2",
CreatedAt: now,
UpdatedAt: now,
Description: "Concurrent bundle 2");
var successCount = 0;
var failureCount = 0;
// Execute concurrently
var task1 = Task.Run(async () =>
{
try
{
await _repository!.CreateBundleAsync(bundle1, cancellationToken);
Interlocked.Increment(ref successCount);
}
catch (PostgresException)
{
Interlocked.Increment(ref failureCount);
}
});
var task2 = Task.Run(async () =>
{
try
{
await _repository!.CreateBundleAsync(bundle2, cancellationToken);
Interlocked.Increment(ref successCount);
}
catch (PostgresException)
{
Interlocked.Increment(ref failureCount);
}
});
await Task.WhenAll(task1, task2);
// Exactly one should succeed, one should fail
Assert.Equal(1, successCount);
Assert.Equal(1, failureCount);
// Verify only one bundle exists
var exists = await _repository!.ExistsAsync(bundleId, tenantId, cancellationToken);
Assert.True(exists);
}
[Fact]
public async Task ConcurrentCreates_DifferentIds_AllSucceed()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundles = Enumerable.Range(1, 5).Select(i =>
{
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
return new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string((char)('a' + i), 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now,
Description: $"Concurrent bundle {i}");
}).ToList();
var successCount = 0;
// Execute all concurrently
var tasks = bundles.Select(async bundle =>
{
await _repository!.CreateBundleAsync(bundle, cancellationToken);
Interlocked.Increment(ref successCount);
});
await Task.WhenAll(tasks);
// All should succeed
Assert.Equal(5, successCount);
// Verify all bundles exist
foreach (var bundle in bundles)
{
var exists = await _repository!.ExistsAsync(bundle.Id, tenantId, cancellationToken);
Assert.True(exists, $"Bundle {bundle.Id} should exist");
}
}
[Fact]
public async Task ConcurrentSealAttempts_SameBundle_AllSucceed()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
await _repository!.CreateBundleAsync(bundle, cancellationToken);
// Multiple concurrent seal attempts (idempotent operation)
var sealTasks = Enumerable.Range(1, 3).Select(async i =>
{
await _repository.MarkBundleSealedAsync(
bundleId,
tenantId,
EvidenceBundleStatus.Sealed,
now.AddMinutes(i),
cancellationToken);
});
// All should complete without throwing
await Task.WhenAll(sealTasks);
// Bundle should be sealed
var fetched = await _repository.GetBundleAsync(bundleId, tenantId, cancellationToken);
Assert.NotNull(fetched);
Assert.Equal(EvidenceBundleStatus.Sealed, fetched.Bundle.Status);
}
// EVIDENCE-5100-003: Same key + different payload → version handling
[Fact]
public async Task SignatureUpsert_SameBundle_UpdatesSignature()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
await _repository!.CreateBundleAsync(bundle, cancellationToken);
// First signature
var signature1 = new EvidenceBundleSignature(
bundleId,
tenantId,
PayloadType: "application/vnd.dsse+json",
Payload: """{"_type":"bundle","bundle_id":"test"}""",
Signature: "sig1",
KeyId: "key1",
Algorithm: "ES256",
Provider: "test",
SignedAt: now);
await _repository.UpsertSignatureAsync(signature1, cancellationToken);
// Verify first signature
var fetchedBefore = await _repository.GetBundleAsync(bundleId, tenantId, cancellationToken);
Assert.NotNull(fetchedBefore?.Signature);
Assert.Equal("sig1", fetchedBefore.Signature.Signature);
Assert.Equal("key1", fetchedBefore.Signature.KeyId);
// Second signature (update)
var signature2 = new EvidenceBundleSignature(
bundleId,
tenantId,
PayloadType: "application/vnd.dsse+json",
Payload: """{"_type":"bundle","bundle_id":"test","version":2}""",
Signature: "sig2",
KeyId: "key2",
Algorithm: "ES256",
Provider: "test",
SignedAt: now.AddMinutes(1));
await _repository.UpsertSignatureAsync(signature2, cancellationToken);
// Verify signature was updated
var fetchedAfter = await _repository.GetBundleAsync(bundleId, tenantId, cancellationToken);
Assert.NotNull(fetchedAfter?.Signature);
Assert.Equal("sig2", fetchedAfter.Signature.Signature);
Assert.Equal("key2", fetchedAfter.Signature.KeyId);
}
[Fact]
public async Task BundleUpdate_AssemblyPhase_UpdatesHashAndStatus()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Pending,
RootHash: new string('0', 64), // Initial placeholder hash
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
await _repository!.CreateBundleAsync(bundle, cancellationToken);
// Update to assembling with new hash
var newHash = new string('a', 64);
await _repository.SetBundleAssemblyAsync(
bundleId,
tenantId,
EvidenceBundleStatus.Assembling,
newHash,
now.AddMinutes(1),
cancellationToken);
// Verify update
var fetched = await _repository.GetBundleAsync(bundleId, tenantId, cancellationToken);
Assert.NotNull(fetched);
Assert.Equal(EvidenceBundleStatus.Assembling, fetched.Bundle.Status);
Assert.Equal(newHash, fetched.Bundle.RootHash);
}
[Fact]
public async Task PortableStorageKey_Update_CreatesVersionedReference()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Export,
EvidenceBundleStatus.Sealed,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
await _repository!.CreateBundleAsync(bundle, cancellationToken);
// No portable storage key initially
var fetchedBefore = await _repository.GetBundleAsync(bundleId, tenantId, cancellationToken);
Assert.NotNull(fetchedBefore);
Assert.Null(fetchedBefore.Bundle.PortableStorageKey);
// Add portable storage key
var portableKey = $"tenants/{tenantId}/portable/{bundleId}/export.zip";
await _repository.UpdatePortableStorageKeyAsync(
bundleId,
tenantId,
portableKey,
now.AddMinutes(1),
cancellationToken);
// Verify portable key was added
var fetchedAfter = await _repository.GetBundleAsync(bundleId, tenantId, cancellationToken);
Assert.NotNull(fetchedAfter);
Assert.Equal(portableKey, fetchedAfter.Bundle.PortableStorageKey);
Assert.NotNull(fetchedAfter.Bundle.PortableGeneratedAt);
}
[Fact]
public async Task Hold_CreateMultiple_AllPersisted()
{
if (_skipReason is not null)
{
Assert.Skip(_skipReason);
}
var cancellationToken = TestContext.Current.CancellationToken;
var tenantId = TenantId.FromGuid(Guid.NewGuid());
var bundleId = EvidenceBundleId.FromGuid(Guid.NewGuid());
var now = DateTimeOffset.UtcNow;
var bundle = new EvidenceBundle(
bundleId,
tenantId,
EvidenceBundleKind.Evaluation,
EvidenceBundleStatus.Sealed,
RootHash: new string('a', 64),
StorageKey: $"tenants/{tenantId}/bundles/{bundleId}/resource",
CreatedAt: now,
UpdatedAt: now);
await _repository!.CreateBundleAsync(bundle, cancellationToken);
// Create multiple holds (versioned/append-only pattern)
var holds = new List<EvidenceHold>();
for (int i = 1; i <= 3; i++)
{
var hold = new EvidenceHold(
EvidenceHoldId.FromGuid(Guid.NewGuid()),
tenantId,
bundleId,
CaseId: $"CASE-{i:D4}",
Reason: $"Legal hold reason {i}",
CreatedAt: now.AddMinutes(i),
ExpiresAt: now.AddDays(30 + i));
var createdHold = await _repository.CreateHoldAsync(hold, cancellationToken);
holds.Add(createdHold);
}
// All holds should be created with unique IDs
Assert.Equal(3, holds.Count);
Assert.True(holds.All(h => h.Id.Value != Guid.Empty));
Assert.True(holds.Select(h => h.Id.Value).Distinct().Count() == 3);
}
public async ValueTask InitializeAsync()
{
try
{
await _postgres.StartAsync();
}
catch (HttpRequestException ex)
{
_skipReason = $"Docker endpoint unavailable: {ex.Message}";
return;
}
catch (DockerApiException ex)
{
_skipReason = $"Docker API error: {ex.Message}";
return;
}
var databaseOptions = new DatabaseOptions
{
ConnectionString = _postgres.ConnectionString,
ApplyMigrationsAtStartup = false
};
_dataSource = new EvidenceLockerDataSource(databaseOptions, NullLogger<EvidenceLockerDataSource>.Instance);
_migrationRunner = new EvidenceLockerMigrationRunner(_dataSource, NullLogger<EvidenceLockerMigrationRunner>.Instance);
// Apply migrations
await _migrationRunner.ApplyAsync(CancellationToken.None);
// Create repository
_repository = new EvidenceBundleRepository(_dataSource);
}
public async ValueTask DisposeAsync()
{
if (_skipReason is not null)
{
return;
}
if (_dataSource is not null)
{
await _dataSource.DisposeAsync();
}
await _postgres.DisposeAsync();
}
}