Add determinism tests for verdict artifact generation and update SHA256 sums script
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,411 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// PolicyQueryDeterminismTests.cs
|
||||
// Sprint: SPRINT_5100_0009_0004_policy_tests
|
||||
// Task: POLICY-5100-009
|
||||
// Description: Model S1 query determinism tests for Policy retrieval ordering
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Policy.Storage.Postgres.Models;
|
||||
using StellaOps.Policy.Storage.Postgres.Repositories;
|
||||
using StellaOps.TestKit;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Policy.Storage.Postgres.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Query determinism tests for Policy storage operations.
|
||||
/// Implements Model S1 (Storage/Postgres) test requirements:
|
||||
/// - Explicit ORDER BY checks for all list queries
|
||||
/// - Same inputs → stable ordering
|
||||
/// - Repeated queries return consistent results
|
||||
/// </summary>
|
||||
[Collection(PolicyPostgresCollection.Name)]
|
||||
[Trait("Category", TestCategories.Integration)]
|
||||
[Trait("Category", "QueryDeterminism")]
|
||||
public sealed class PolicyQueryDeterminismTests : IAsyncLifetime
|
||||
{
|
||||
private readonly PolicyPostgresFixture _fixture;
|
||||
private PolicyDataSource _dataSource = null!;
|
||||
private PackRepository _packRepository = null!;
|
||||
private PackVersionRepository _packVersionRepository = null!;
|
||||
private RiskProfileRepository _riskProfileRepository = null!;
|
||||
private RuleRepository _ruleRepository = null!;
|
||||
private PolicyAuditRepository _auditRepository = null!;
|
||||
private readonly string _tenantId = Guid.NewGuid().ToString();
|
||||
|
||||
public PolicyQueryDeterminismTests(PolicyPostgresFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
await _fixture.TruncateAllTablesAsync();
|
||||
|
||||
var options = _fixture.Fixture.CreateOptions();
|
||||
options.SchemaName = _fixture.SchemaName;
|
||||
_dataSource = new PolicyDataSource(Options.Create(options), NullLogger<PolicyDataSource>.Instance);
|
||||
_packRepository = new PackRepository(_dataSource, NullLogger<PackRepository>.Instance);
|
||||
_packVersionRepository = new PackVersionRepository(_dataSource, NullLogger<PackVersionRepository>.Instance);
|
||||
_riskProfileRepository = new RiskProfileRepository(_dataSource, NullLogger<RiskProfileRepository>.Instance);
|
||||
_ruleRepository = new RuleRepository(_dataSource, NullLogger<RuleRepository>.Instance);
|
||||
_auditRepository = new PolicyAuditRepository(_dataSource, NullLogger<PolicyAuditRepository>.Instance);
|
||||
}
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
|
||||
[Fact]
|
||||
public async Task GetAllPacks_MultipleQueries_ReturnsDeterministicOrder()
|
||||
{
|
||||
// Arrange
|
||||
var packs = new[]
|
||||
{
|
||||
await CreatePackAsync("pack-c"),
|
||||
await CreatePackAsync("pack-a"),
|
||||
await CreatePackAsync("pack-b"),
|
||||
await CreatePackAsync("pack-e"),
|
||||
await CreatePackAsync("pack-d")
|
||||
};
|
||||
|
||||
// Act - Query multiple times
|
||||
var results1 = await _packRepository.GetAllAsync(_tenantId);
|
||||
var results2 = await _packRepository.GetAllAsync(_tenantId);
|
||||
var results3 = await _packRepository.GetAllAsync(_tenantId);
|
||||
|
||||
// Assert - All queries should return same order
|
||||
var ids1 = results1.Select(p => p.Id).ToList();
|
||||
var ids2 = results2.Select(p => p.Id).ToList();
|
||||
var ids3 = results3.Select(p => p.Id).ToList();
|
||||
|
||||
ids1.Should().Equal(ids2);
|
||||
ids2.Should().Equal(ids3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetPackVersions_MultipleQueries_ReturnsDeterministicOrder()
|
||||
{
|
||||
// Arrange
|
||||
var pack = await CreatePackAsync("version-order-test");
|
||||
|
||||
// Create versions in non-sequential order
|
||||
await CreatePackVersionAsync(pack.Id, 3, publish: true);
|
||||
await CreatePackVersionAsync(pack.Id, 1, publish: true);
|
||||
await CreatePackVersionAsync(pack.Id, 5, publish: true);
|
||||
await CreatePackVersionAsync(pack.Id, 2, publish: true);
|
||||
await CreatePackVersionAsync(pack.Id, 4, publish: true);
|
||||
|
||||
// Act - Query multiple times
|
||||
var results1 = await _packVersionRepository.GetByPackIdAsync(pack.Id, publishedOnly: true);
|
||||
var results2 = await _packVersionRepository.GetByPackIdAsync(pack.Id, publishedOnly: true);
|
||||
var results3 = await _packVersionRepository.GetByPackIdAsync(pack.Id, publishedOnly: true);
|
||||
|
||||
// Assert - All queries should return same order
|
||||
var versions1 = results1.Select(v => v.Version).ToList();
|
||||
var versions2 = results2.Select(v => v.Version).ToList();
|
||||
var versions3 = results3.Select(v => v.Version).ToList();
|
||||
|
||||
versions1.Should().Equal(versions2);
|
||||
versions2.Should().Equal(versions3);
|
||||
|
||||
// Should be ordered by version descending (newest first)
|
||||
versions1.Should().BeInDescendingOrder();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetRiskProfiles_MultipleQueries_ReturnsDeterministicOrder()
|
||||
{
|
||||
// Arrange
|
||||
var profiles = new[]
|
||||
{
|
||||
await CreateRiskProfileAsync("profile-z"),
|
||||
await CreateRiskProfileAsync("profile-a"),
|
||||
await CreateRiskProfileAsync("profile-m"),
|
||||
await CreateRiskProfileAsync("profile-b"),
|
||||
await CreateRiskProfileAsync("profile-y")
|
||||
};
|
||||
|
||||
// Act - Query multiple times
|
||||
var results1 = await _riskProfileRepository.GetAllAsync(_tenantId);
|
||||
var results2 = await _riskProfileRepository.GetAllAsync(_tenantId);
|
||||
var results3 = await _riskProfileRepository.GetAllAsync(_tenantId);
|
||||
|
||||
// Assert - All queries should return same order
|
||||
var ids1 = results1.Select(p => p.Id).ToList();
|
||||
var ids2 = results2.Select(p => p.Id).ToList();
|
||||
var ids3 = results3.Select(p => p.Id).ToList();
|
||||
|
||||
ids1.Should().Equal(ids2);
|
||||
ids2.Should().Equal(ids3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetRules_MultipleQueries_ReturnsDeterministicOrder()
|
||||
{
|
||||
// Arrange
|
||||
var pack = await CreatePackAsync("rules-order-test");
|
||||
var version = await CreatePackVersionAsync(pack.Id, 1, publish: true);
|
||||
|
||||
var rules = new[]
|
||||
{
|
||||
await CreateRuleAsync(version.Id, "rule-zebra"),
|
||||
await CreateRuleAsync(version.Id, "rule-alpha"),
|
||||
await CreateRuleAsync(version.Id, "rule-gamma"),
|
||||
await CreateRuleAsync(version.Id, "rule-beta"),
|
||||
await CreateRuleAsync(version.Id, "rule-delta")
|
||||
};
|
||||
|
||||
// Act - Query multiple times
|
||||
var results1 = await _ruleRepository.GetByVersionIdAsync(version.Id);
|
||||
var results2 = await _ruleRepository.GetByVersionIdAsync(version.Id);
|
||||
var results3 = await _ruleRepository.GetByVersionIdAsync(version.Id);
|
||||
|
||||
// Assert - All queries should return same order
|
||||
var ids1 = results1.Select(r => r.Id).ToList();
|
||||
var ids2 = results2.Select(r => r.Id).ToList();
|
||||
var ids3 = results3.Select(r => r.Id).ToList();
|
||||
|
||||
ids1.Should().Equal(ids2);
|
||||
ids2.Should().Equal(ids3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetAuditEntries_MultipleQueries_ReturnsDeterministicOrder()
|
||||
{
|
||||
// Arrange
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
await CreateAuditEntryAsync($"action-{i}");
|
||||
}
|
||||
|
||||
// Act - Query multiple times
|
||||
var results1 = await _auditRepository.GetRecentAsync(_tenantId, 10);
|
||||
var results2 = await _auditRepository.GetRecentAsync(_tenantId, 10);
|
||||
var results3 = await _auditRepository.GetRecentAsync(_tenantId, 10);
|
||||
|
||||
// Assert - All queries should return same order
|
||||
var ids1 = results1.Select(a => a.Id).ToList();
|
||||
var ids2 = results2.Select(a => a.Id).ToList();
|
||||
var ids3 = results3.Select(a => a.Id).ToList();
|
||||
|
||||
ids1.Should().Equal(ids2);
|
||||
ids2.Should().Equal(ids3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentQueries_SamePack_AllReturnIdenticalResults()
|
||||
{
|
||||
// Arrange
|
||||
var pack = await CreatePackAsync("concurrent-test");
|
||||
await CreatePackVersionAsync(pack.Id, 1, publish: true);
|
||||
await CreatePackVersionAsync(pack.Id, 2, publish: true);
|
||||
|
||||
// Act - 20 concurrent queries
|
||||
var tasks = Enumerable.Range(0, 20)
|
||||
.Select(_ => _packVersionRepository.GetByPackIdAsync(pack.Id, publishedOnly: true))
|
||||
.ToList();
|
||||
|
||||
var results = await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - All should return identical order
|
||||
var firstOrder = results[0].Select(v => v.Version).ToList();
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Select(v => v.Version).ToList().Should().Equal(firstOrder);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetLatestVersion_MultipleQueries_ReturnsConsistentResult()
|
||||
{
|
||||
// Arrange
|
||||
var pack = await CreatePackAsync("latest-consistent-test");
|
||||
await CreatePackVersionAsync(pack.Id, 1, publish: true);
|
||||
await CreatePackVersionAsync(pack.Id, 2, publish: true);
|
||||
await CreatePackVersionAsync(pack.Id, 3, publish: true);
|
||||
|
||||
// Act - Query multiple times
|
||||
var results = new List<PackVersionEntity?>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(await _packVersionRepository.GetLatestAsync(pack.Id));
|
||||
}
|
||||
|
||||
// Assert - All should return version 3
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Should().NotBeNull();
|
||||
r!.Version.Should().Be(3);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetById_MultipleQueries_ReturnsConsistentResult()
|
||||
{
|
||||
// Arrange
|
||||
var pack = await CreatePackAsync("get-by-id-test");
|
||||
|
||||
// Act - Query multiple times
|
||||
var results = new List<PackEntity?>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(await _packRepository.GetByIdAsync(_tenantId, pack.Id));
|
||||
}
|
||||
|
||||
// Assert - All should return identical pack
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Should().NotBeNull();
|
||||
r!.Id.Should().Be(pack.Id);
|
||||
r.Name.Should().Be("get-by-id-test");
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetByName_MultipleQueries_ReturnsConsistentResult()
|
||||
{
|
||||
// Arrange
|
||||
var pack = await CreatePackAsync("name-lookup-test");
|
||||
|
||||
// Act - Query multiple times
|
||||
var results = new List<PackEntity?>();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
results.Add(await _packRepository.GetByNameAsync(_tenantId, "name-lookup-test"));
|
||||
}
|
||||
|
||||
// Assert - All should return same pack
|
||||
results.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Should().NotBeNull();
|
||||
r!.Id.Should().Be(pack.Id);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EmptyTenant_GetAllPacks_ReturnsEmptyConsistently()
|
||||
{
|
||||
// Arrange
|
||||
var emptyTenantId = Guid.NewGuid().ToString();
|
||||
|
||||
// Act - Query empty tenant multiple times
|
||||
var results = new List<IReadOnlyList<PackEntity>>();
|
||||
for (int i = 0; i < 5; i++)
|
||||
{
|
||||
results.Add(await _packRepository.GetAllAsync(emptyTenantId));
|
||||
}
|
||||
|
||||
// Assert - All should return empty
|
||||
results.Should().AllSatisfy(r => r.Should().BeEmpty());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TenantIsolation_PacksInDifferentTenants_QueriesReturnOnlyOwnTenant()
|
||||
{
|
||||
// Arrange
|
||||
var tenant1 = Guid.NewGuid().ToString();
|
||||
var tenant2 = Guid.NewGuid().ToString();
|
||||
|
||||
var pack1 = await CreatePackAsync("tenant1-pack", tenant1);
|
||||
var pack2 = await CreatePackAsync("tenant2-pack", tenant2);
|
||||
|
||||
// Act
|
||||
var tenant1Packs = await _packRepository.GetAllAsync(tenant1);
|
||||
var tenant2Packs = await _packRepository.GetAllAsync(tenant2);
|
||||
|
||||
// Assert
|
||||
tenant1Packs.Should().HaveCount(1);
|
||||
tenant1Packs[0].Id.Should().Be(pack1.Id);
|
||||
|
||||
tenant2Packs.Should().HaveCount(1);
|
||||
tenant2Packs[0].Id.Should().Be(pack2.Id);
|
||||
}
|
||||
|
||||
private async Task<PackEntity> CreatePackAsync(string name, string? tenantId = null)
|
||||
{
|
||||
var pack = new PackEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TenantId = tenantId ?? _tenantId,
|
||||
Name = name,
|
||||
DisplayName = $"Display {name}",
|
||||
IsBuiltin = false
|
||||
};
|
||||
await _packRepository.CreateAsync(pack);
|
||||
return pack;
|
||||
}
|
||||
|
||||
private async Task<PackVersionEntity> CreatePackVersionAsync(Guid packId, int version, bool publish = false)
|
||||
{
|
||||
var packVersion = new PackVersionEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
PackId = packId,
|
||||
Version = version,
|
||||
Description = $"Version {version}",
|
||||
RulesHash = $"rules-hash-{version}-{Guid.NewGuid():N}",
|
||||
IsPublished = false
|
||||
};
|
||||
|
||||
var created = await _packVersionRepository.CreateAsync(packVersion);
|
||||
|
||||
if (publish)
|
||||
{
|
||||
await _packVersionRepository.PublishAsync(created.Id, "test-publisher");
|
||||
created = (await _packVersionRepository.GetByIdAsync(created.Id))!;
|
||||
}
|
||||
|
||||
return created;
|
||||
}
|
||||
|
||||
private async Task<RiskProfileEntity> CreateRiskProfileAsync(string name)
|
||||
{
|
||||
var profile = new RiskProfileEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TenantId = _tenantId,
|
||||
Name = name,
|
||||
DisplayName = $"Display {name}",
|
||||
Version = 1,
|
||||
PolicyContent = """{"rules": []}""",
|
||||
ContentHash = $"hash-{Guid.NewGuid():N}"
|
||||
};
|
||||
await _riskProfileRepository.CreateAsync(profile);
|
||||
return profile;
|
||||
}
|
||||
|
||||
private async Task<RuleEntity> CreateRuleAsync(Guid versionId, string name)
|
||||
{
|
||||
var rule = new RuleEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
PackVersionId = versionId,
|
||||
Name = name,
|
||||
DisplayName = $"Display {name}",
|
||||
Severity = "HIGH",
|
||||
RuleContent = """{"condition": "always"}""",
|
||||
ContentHash = $"hash-{Guid.NewGuid():N}"
|
||||
};
|
||||
await _ruleRepository.CreateAsync(rule);
|
||||
return rule;
|
||||
}
|
||||
|
||||
private async Task<PolicyAuditEntity> CreateAuditEntryAsync(string action)
|
||||
{
|
||||
var audit = new PolicyAuditEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TenantId = _tenantId,
|
||||
Action = action,
|
||||
Actor = "test-user",
|
||||
EntityType = "Pack",
|
||||
EntityId = Guid.NewGuid().ToString(),
|
||||
Timestamp = DateTimeOffset.UtcNow,
|
||||
Details = """{"test": true}"""
|
||||
};
|
||||
await _auditRepository.CreateAsync(audit);
|
||||
return audit;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user