7503c19b8f
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
412 lines
14 KiB
C#
412 lines
14 KiB
C#
// -----------------------------------------------------------------------------
|
|
// PolicyQueryDeterminismTests.cs
|
|
// Sprint: SPRINT_5100_0009_0004_policy_tests
|
|
// Task: POLICY-5100-009
|
|
// Description: Model S1 query determinism tests for Policy retrieval ordering
|
|
// -----------------------------------------------------------------------------
|
|
|
|
using FluentAssertions;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using StellaOps.Policy.Storage.Postgres.Models;
|
|
using StellaOps.Policy.Storage.Postgres.Repositories;
|
|
using StellaOps.TestKit;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Policy.Storage.Postgres.Tests;
|
|
|
|
/// <summary>
|
|
/// Query determinism tests for Policy storage operations.
|
|
/// Implements Model S1 (Storage/Postgres) test requirements:
|
|
/// - Explicit ORDER BY checks for all list queries
|
|
/// - Same inputs → stable ordering
|
|
/// - Repeated queries return consistent results
|
|
/// </summary>
|
|
[Collection(PolicyPostgresCollection.Name)]
|
|
[Trait("Category", TestCategories.Integration)]
|
|
[Trait("Category", "QueryDeterminism")]
|
|
public sealed class PolicyQueryDeterminismTests : IAsyncLifetime
|
|
{
|
|
private readonly PolicyPostgresFixture _fixture;
|
|
private PolicyDataSource _dataSource = null!;
|
|
private PackRepository _packRepository = null!;
|
|
private PackVersionRepository _packVersionRepository = null!;
|
|
private RiskProfileRepository _riskProfileRepository = null!;
|
|
private RuleRepository _ruleRepository = null!;
|
|
private PolicyAuditRepository _auditRepository = null!;
|
|
private readonly string _tenantId = Guid.NewGuid().ToString();
|
|
|
|
public PolicyQueryDeterminismTests(PolicyPostgresFixture fixture)
|
|
{
|
|
_fixture = fixture;
|
|
}
|
|
|
|
public async Task InitializeAsync()
|
|
{
|
|
await _fixture.TruncateAllTablesAsync();
|
|
|
|
var options = _fixture.Fixture.CreateOptions();
|
|
options.SchemaName = _fixture.SchemaName;
|
|
_dataSource = new PolicyDataSource(Options.Create(options), NullLogger<PolicyDataSource>.Instance);
|
|
_packRepository = new PackRepository(_dataSource, NullLogger<PackRepository>.Instance);
|
|
_packVersionRepository = new PackVersionRepository(_dataSource, NullLogger<PackVersionRepository>.Instance);
|
|
_riskProfileRepository = new RiskProfileRepository(_dataSource, NullLogger<RiskProfileRepository>.Instance);
|
|
_ruleRepository = new RuleRepository(_dataSource, NullLogger<RuleRepository>.Instance);
|
|
_auditRepository = new PolicyAuditRepository(_dataSource, NullLogger<PolicyAuditRepository>.Instance);
|
|
}
|
|
|
|
public Task DisposeAsync() => Task.CompletedTask;
|
|
|
|
[Fact]
|
|
public async Task GetAllPacks_MultipleQueries_ReturnsDeterministicOrder()
|
|
{
|
|
// Arrange
|
|
var packs = new[]
|
|
{
|
|
await CreatePackAsync("pack-c"),
|
|
await CreatePackAsync("pack-a"),
|
|
await CreatePackAsync("pack-b"),
|
|
await CreatePackAsync("pack-e"),
|
|
await CreatePackAsync("pack-d")
|
|
};
|
|
|
|
// Act - Query multiple times
|
|
var results1 = await _packRepository.GetAllAsync(_tenantId);
|
|
var results2 = await _packRepository.GetAllAsync(_tenantId);
|
|
var results3 = await _packRepository.GetAllAsync(_tenantId);
|
|
|
|
// Assert - All queries should return same order
|
|
var ids1 = results1.Select(p => p.Id).ToList();
|
|
var ids2 = results2.Select(p => p.Id).ToList();
|
|
var ids3 = results3.Select(p => p.Id).ToList();
|
|
|
|
ids1.Should().Equal(ids2);
|
|
ids2.Should().Equal(ids3);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetPackVersions_MultipleQueries_ReturnsDeterministicOrder()
|
|
{
|
|
// Arrange
|
|
var pack = await CreatePackAsync("version-order-test");
|
|
|
|
// Create versions in non-sequential order
|
|
await CreatePackVersionAsync(pack.Id, 3, publish: true);
|
|
await CreatePackVersionAsync(pack.Id, 1, publish: true);
|
|
await CreatePackVersionAsync(pack.Id, 5, publish: true);
|
|
await CreatePackVersionAsync(pack.Id, 2, publish: true);
|
|
await CreatePackVersionAsync(pack.Id, 4, publish: true);
|
|
|
|
// Act - Query multiple times
|
|
var results1 = await _packVersionRepository.GetByPackIdAsync(pack.Id, publishedOnly: true);
|
|
var results2 = await _packVersionRepository.GetByPackIdAsync(pack.Id, publishedOnly: true);
|
|
var results3 = await _packVersionRepository.GetByPackIdAsync(pack.Id, publishedOnly: true);
|
|
|
|
// Assert - All queries should return same order
|
|
var versions1 = results1.Select(v => v.Version).ToList();
|
|
var versions2 = results2.Select(v => v.Version).ToList();
|
|
var versions3 = results3.Select(v => v.Version).ToList();
|
|
|
|
versions1.Should().Equal(versions2);
|
|
versions2.Should().Equal(versions3);
|
|
|
|
// Should be ordered by version descending (newest first)
|
|
versions1.Should().BeInDescendingOrder();
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetRiskProfiles_MultipleQueries_ReturnsDeterministicOrder()
|
|
{
|
|
// Arrange
|
|
var profiles = new[]
|
|
{
|
|
await CreateRiskProfileAsync("profile-z"),
|
|
await CreateRiskProfileAsync("profile-a"),
|
|
await CreateRiskProfileAsync("profile-m"),
|
|
await CreateRiskProfileAsync("profile-b"),
|
|
await CreateRiskProfileAsync("profile-y")
|
|
};
|
|
|
|
// Act - Query multiple times
|
|
var results1 = await _riskProfileRepository.GetAllAsync(_tenantId);
|
|
var results2 = await _riskProfileRepository.GetAllAsync(_tenantId);
|
|
var results3 = await _riskProfileRepository.GetAllAsync(_tenantId);
|
|
|
|
// Assert - All queries should return same order
|
|
var ids1 = results1.Select(p => p.Id).ToList();
|
|
var ids2 = results2.Select(p => p.Id).ToList();
|
|
var ids3 = results3.Select(p => p.Id).ToList();
|
|
|
|
ids1.Should().Equal(ids2);
|
|
ids2.Should().Equal(ids3);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetRules_MultipleQueries_ReturnsDeterministicOrder()
|
|
{
|
|
// Arrange
|
|
var pack = await CreatePackAsync("rules-order-test");
|
|
var version = await CreatePackVersionAsync(pack.Id, 1, publish: true);
|
|
|
|
var rules = new[]
|
|
{
|
|
await CreateRuleAsync(version.Id, "rule-zebra"),
|
|
await CreateRuleAsync(version.Id, "rule-alpha"),
|
|
await CreateRuleAsync(version.Id, "rule-gamma"),
|
|
await CreateRuleAsync(version.Id, "rule-beta"),
|
|
await CreateRuleAsync(version.Id, "rule-delta")
|
|
};
|
|
|
|
// Act - Query multiple times
|
|
var results1 = await _ruleRepository.GetByVersionIdAsync(version.Id);
|
|
var results2 = await _ruleRepository.GetByVersionIdAsync(version.Id);
|
|
var results3 = await _ruleRepository.GetByVersionIdAsync(version.Id);
|
|
|
|
// Assert - All queries should return same order
|
|
var ids1 = results1.Select(r => r.Id).ToList();
|
|
var ids2 = results2.Select(r => r.Id).ToList();
|
|
var ids3 = results3.Select(r => r.Id).ToList();
|
|
|
|
ids1.Should().Equal(ids2);
|
|
ids2.Should().Equal(ids3);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetAuditEntries_MultipleQueries_ReturnsDeterministicOrder()
|
|
{
|
|
// Arrange
|
|
for (int i = 0; i < 5; i++)
|
|
{
|
|
await CreateAuditEntryAsync($"action-{i}");
|
|
}
|
|
|
|
// Act - Query multiple times
|
|
var results1 = await _auditRepository.GetRecentAsync(_tenantId, 10);
|
|
var results2 = await _auditRepository.GetRecentAsync(_tenantId, 10);
|
|
var results3 = await _auditRepository.GetRecentAsync(_tenantId, 10);
|
|
|
|
// Assert - All queries should return same order
|
|
var ids1 = results1.Select(a => a.Id).ToList();
|
|
var ids2 = results2.Select(a => a.Id).ToList();
|
|
var ids3 = results3.Select(a => a.Id).ToList();
|
|
|
|
ids1.Should().Equal(ids2);
|
|
ids2.Should().Equal(ids3);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ConcurrentQueries_SamePack_AllReturnIdenticalResults()
|
|
{
|
|
// Arrange
|
|
var pack = await CreatePackAsync("concurrent-test");
|
|
await CreatePackVersionAsync(pack.Id, 1, publish: true);
|
|
await CreatePackVersionAsync(pack.Id, 2, publish: true);
|
|
|
|
// Act - 20 concurrent queries
|
|
var tasks = Enumerable.Range(0, 20)
|
|
.Select(_ => _packVersionRepository.GetByPackIdAsync(pack.Id, publishedOnly: true))
|
|
.ToList();
|
|
|
|
var results = await Task.WhenAll(tasks);
|
|
|
|
// Assert - All should return identical order
|
|
var firstOrder = results[0].Select(v => v.Version).ToList();
|
|
results.Should().AllSatisfy(r =>
|
|
{
|
|
r.Select(v => v.Version).ToList().Should().Equal(firstOrder);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetLatestVersion_MultipleQueries_ReturnsConsistentResult()
|
|
{
|
|
// Arrange
|
|
var pack = await CreatePackAsync("latest-consistent-test");
|
|
await CreatePackVersionAsync(pack.Id, 1, publish: true);
|
|
await CreatePackVersionAsync(pack.Id, 2, publish: true);
|
|
await CreatePackVersionAsync(pack.Id, 3, publish: true);
|
|
|
|
// Act - Query multiple times
|
|
var results = new List<PackVersionEntity?>();
|
|
for (int i = 0; i < 10; i++)
|
|
{
|
|
results.Add(await _packVersionRepository.GetLatestAsync(pack.Id));
|
|
}
|
|
|
|
// Assert - All should return version 3
|
|
results.Should().AllSatisfy(r =>
|
|
{
|
|
r.Should().NotBeNull();
|
|
r!.Version.Should().Be(3);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetById_MultipleQueries_ReturnsConsistentResult()
|
|
{
|
|
// Arrange
|
|
var pack = await CreatePackAsync("get-by-id-test");
|
|
|
|
// Act - Query multiple times
|
|
var results = new List<PackEntity?>();
|
|
for (int i = 0; i < 10; i++)
|
|
{
|
|
results.Add(await _packRepository.GetByIdAsync(_tenantId, pack.Id));
|
|
}
|
|
|
|
// Assert - All should return identical pack
|
|
results.Should().AllSatisfy(r =>
|
|
{
|
|
r.Should().NotBeNull();
|
|
r!.Id.Should().Be(pack.Id);
|
|
r.Name.Should().Be("get-by-id-test");
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task GetByName_MultipleQueries_ReturnsConsistentResult()
|
|
{
|
|
// Arrange
|
|
var pack = await CreatePackAsync("name-lookup-test");
|
|
|
|
// Act - Query multiple times
|
|
var results = new List<PackEntity?>();
|
|
for (int i = 0; i < 10; i++)
|
|
{
|
|
results.Add(await _packRepository.GetByNameAsync(_tenantId, "name-lookup-test"));
|
|
}
|
|
|
|
// Assert - All should return same pack
|
|
results.Should().AllSatisfy(r =>
|
|
{
|
|
r.Should().NotBeNull();
|
|
r!.Id.Should().Be(pack.Id);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public async Task EmptyTenant_GetAllPacks_ReturnsEmptyConsistently()
|
|
{
|
|
// Arrange
|
|
var emptyTenantId = Guid.NewGuid().ToString();
|
|
|
|
// Act - Query empty tenant multiple times
|
|
var results = new List<IReadOnlyList<PackEntity>>();
|
|
for (int i = 0; i < 5; i++)
|
|
{
|
|
results.Add(await _packRepository.GetAllAsync(emptyTenantId));
|
|
}
|
|
|
|
// Assert - All should return empty
|
|
results.Should().AllSatisfy(r => r.Should().BeEmpty());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task TenantIsolation_PacksInDifferentTenants_QueriesReturnOnlyOwnTenant()
|
|
{
|
|
// Arrange
|
|
var tenant1 = Guid.NewGuid().ToString();
|
|
var tenant2 = Guid.NewGuid().ToString();
|
|
|
|
var pack1 = await CreatePackAsync("tenant1-pack", tenant1);
|
|
var pack2 = await CreatePackAsync("tenant2-pack", tenant2);
|
|
|
|
// Act
|
|
var tenant1Packs = await _packRepository.GetAllAsync(tenant1);
|
|
var tenant2Packs = await _packRepository.GetAllAsync(tenant2);
|
|
|
|
// Assert
|
|
tenant1Packs.Should().HaveCount(1);
|
|
tenant1Packs[0].Id.Should().Be(pack1.Id);
|
|
|
|
tenant2Packs.Should().HaveCount(1);
|
|
tenant2Packs[0].Id.Should().Be(pack2.Id);
|
|
}
|
|
|
|
private async Task<PackEntity> CreatePackAsync(string name, string? tenantId = null)
|
|
{
|
|
var pack = new PackEntity
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
TenantId = tenantId ?? _tenantId,
|
|
Name = name,
|
|
DisplayName = $"Display {name}",
|
|
IsBuiltin = false
|
|
};
|
|
await _packRepository.CreateAsync(pack);
|
|
return pack;
|
|
}
|
|
|
|
private async Task<PackVersionEntity> CreatePackVersionAsync(Guid packId, int version, bool publish = false)
|
|
{
|
|
var packVersion = new PackVersionEntity
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
PackId = packId,
|
|
Version = version,
|
|
Description = $"Version {version}",
|
|
RulesHash = $"rules-hash-{version}-{Guid.NewGuid():N}",
|
|
IsPublished = false
|
|
};
|
|
|
|
var created = await _packVersionRepository.CreateAsync(packVersion);
|
|
|
|
if (publish)
|
|
{
|
|
await _packVersionRepository.PublishAsync(created.Id, "test-publisher");
|
|
created = (await _packVersionRepository.GetByIdAsync(created.Id))!;
|
|
}
|
|
|
|
return created;
|
|
}
|
|
|
|
private async Task<RiskProfileEntity> CreateRiskProfileAsync(string name)
|
|
{
|
|
var profile = new RiskProfileEntity
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
TenantId = _tenantId,
|
|
Name = name,
|
|
DisplayName = $"Display {name}",
|
|
Version = 1,
|
|
PolicyContent = """{"rules": []}""",
|
|
ContentHash = $"hash-{Guid.NewGuid():N}"
|
|
};
|
|
await _riskProfileRepository.CreateAsync(profile);
|
|
return profile;
|
|
}
|
|
|
|
private async Task<RuleEntity> CreateRuleAsync(Guid versionId, string name)
|
|
{
|
|
var rule = new RuleEntity
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
PackVersionId = versionId,
|
|
Name = name,
|
|
DisplayName = $"Display {name}",
|
|
Severity = "HIGH",
|
|
RuleContent = """{"condition": "always"}""",
|
|
ContentHash = $"hash-{Guid.NewGuid():N}"
|
|
};
|
|
await _ruleRepository.CreateAsync(rule);
|
|
return rule;
|
|
}
|
|
|
|
private async Task<PolicyAuditEntity> CreateAuditEntryAsync(string action)
|
|
{
|
|
var audit = new PolicyAuditEntity
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
TenantId = _tenantId,
|
|
Action = action,
|
|
Actor = "test-user",
|
|
EntityType = "Pack",
|
|
EntityId = Guid.NewGuid().ToString(),
|
|
Timestamp = DateTimeOffset.UtcNow,
|
|
Details = """{"test": true}"""
|
|
};
|
|
await _auditRepository.CreateAsync(audit);
|
|
return audit;
|
|
}
|
|
}
|