Add determinism tests for verdict artifact generation and update SHA256 sums script
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,191 @@
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TestKit.Connectors;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for connector live schema drift detection tests.
|
||||
/// These tests fetch from live upstream sources and compare against stored fixtures.
|
||||
///
|
||||
/// IMPORTANT: These tests are opt-in and disabled by default.
|
||||
/// To enable: set STELLAOPS_LIVE_TESTS=true environment variable.
|
||||
/// To also auto-update fixtures on drift: set STELLAOPS_UPDATE_FIXTURES=true.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Usage:
|
||||
/// <code>
|
||||
/// [Trait("Category", TestCategories.Live)]
|
||||
/// public class NvdLiveSchemaTests : ConnectorLiveSchemaTestBase
|
||||
/// {
|
||||
/// protected override string FixturesDirectory => "Fixtures";
|
||||
/// protected override string ConnectorName => "NVD";
|
||||
///
|
||||
/// protected override IEnumerable<LiveSchemaTestCase> GetTestCases()
|
||||
/// {
|
||||
/// yield return new("typical-cve.json", "https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=CVE-2024-0001");
|
||||
/// }
|
||||
///
|
||||
/// [Fact]
|
||||
/// public Task DetectSchemaDrift() => RunSchemaDriftTestsAsync();
|
||||
/// }
|
||||
/// </code>
|
||||
/// </remarks>
|
||||
public abstract class ConnectorLiveSchemaTestBase : IAsyncLifetime
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly FixtureUpdater _fixtureUpdater;
|
||||
private readonly List<FixtureDriftReport> _driftReports = new();
|
||||
|
||||
protected ConnectorLiveSchemaTestBase()
|
||||
{
|
||||
_httpClient = new HttpClient
|
||||
{
|
||||
Timeout = TimeSpan.FromSeconds(30)
|
||||
};
|
||||
_fixtureUpdater = new FixtureUpdater(FixturesDirectory, _httpClient);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the base directory for test fixtures (relative to test assembly).
|
||||
/// </summary>
|
||||
protected abstract string FixturesDirectory { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the connector name for logging purposes.
|
||||
/// </summary>
|
||||
protected abstract string ConnectorName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the test cases mapping fixtures to live URLs.
|
||||
/// </summary>
|
||||
protected abstract IEnumerable<LiveSchemaTestCase> GetTestCases();
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if live tests are enabled.
|
||||
/// </summary>
|
||||
protected static bool IsEnabled =>
|
||||
Environment.GetEnvironmentVariable("STELLAOPS_LIVE_TESTS") == "true";
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if fixture auto-update is enabled.
|
||||
/// </summary>
|
||||
protected static bool IsAutoUpdateEnabled =>
|
||||
Environment.GetEnvironmentVariable("STELLAOPS_UPDATE_FIXTURES") == "true";
|
||||
|
||||
/// <summary>
|
||||
/// Optional request headers for live requests.
|
||||
/// </summary>
|
||||
protected virtual Dictionary<string, string> RequestHeaders => new();
|
||||
|
||||
/// <summary>
|
||||
/// Runs all schema drift tests for this connector.
|
||||
/// </summary>
|
||||
protected async Task RunSchemaDriftTestsAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
// Skip test when not explicitly enabled
|
||||
return;
|
||||
}
|
||||
|
||||
var testCases = GetTestCases().ToList();
|
||||
_driftReports.Clear();
|
||||
|
||||
foreach (var testCase in testCases)
|
||||
{
|
||||
var report = await _fixtureUpdater.CheckDriftAsync(testCase.LiveUrl, testCase.FixtureName, ct);
|
||||
_driftReports.Add(report);
|
||||
|
||||
if (report.HasDrift && IsAutoUpdateEnabled && !string.IsNullOrEmpty(report.LiveContent))
|
||||
{
|
||||
await _fixtureUpdater.UpdateJsonFixtureFromUrlAsync(testCase.LiveUrl, testCase.FixtureName, ct);
|
||||
}
|
||||
}
|
||||
|
||||
// Report all drift findings
|
||||
var driftCount = _driftReports.Count(r => r.HasDrift);
|
||||
if (driftCount > 0)
|
||||
{
|
||||
var driftMessages = _driftReports
|
||||
.Where(r => r.HasDrift)
|
||||
.Select(r => $" - {r.FixtureName}: {r.Message}");
|
||||
|
||||
var message = $"Schema drift detected in {driftCount}/{_driftReports.Count} fixtures for {ConnectorName}:\n" +
|
||||
string.Join("\n", driftMessages);
|
||||
|
||||
if (IsAutoUpdateEnabled)
|
||||
{
|
||||
// Warn but don't fail when auto-updating
|
||||
Console.WriteLine($"[WARN] {message}");
|
||||
Console.WriteLine($"[INFO] Fixtures have been auto-updated. Review changes before committing.");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fail test when drift detected without auto-update
|
||||
Assert.Fail(message + "\n\nSet STELLAOPS_UPDATE_FIXTURES=true to auto-update fixtures.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the drift reports from the last test run.
|
||||
/// </summary>
|
||||
public IReadOnlyList<FixtureDriftReport> DriftReports => _driftReports.AsReadOnly();
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
foreach (var (key, value) in RequestHeaders)
|
||||
{
|
||||
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation(key, value);
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_httpClient.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test case for live schema drift detection.
|
||||
/// </summary>
|
||||
/// <param name="FixtureName">The fixture file name in the fixtures directory.</param>
|
||||
/// <param name="LiveUrl">The live URL to fetch for comparison.</param>
|
||||
/// <param name="Description">Optional description of what this fixture tests.</param>
|
||||
public sealed record LiveSchemaTestCase(
|
||||
string FixtureName,
|
||||
string LiveUrl,
|
||||
string? Description = null);
|
||||
|
||||
/// <summary>
|
||||
/// Attribute to mark tests that require live external services.
|
||||
/// These tests are skipped unless STELLAOPS_LIVE_TESTS=true.
|
||||
/// </summary>
|
||||
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false)]
|
||||
public sealed class LiveTestAttribute : FactAttribute
|
||||
{
|
||||
public LiveTestAttribute()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("STELLAOPS_LIVE_TESTS") != "true")
|
||||
{
|
||||
Skip = "Live tests are disabled. Set STELLAOPS_LIVE_TESTS=true to enable.";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Theory attribute for live tests that are skipped unless explicitly enabled.
|
||||
/// </summary>
|
||||
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
|
||||
public sealed class LiveTheoryAttribute : TheoryAttribute
|
||||
{
|
||||
public LiveTheoryAttribute()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("STELLAOPS_LIVE_TESTS") != "true")
|
||||
{
|
||||
Skip = "Live tests are disabled. Set STELLAOPS_LIVE_TESTS=true to enable.";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ConnectorSecurityTestBase.cs
|
||||
// Sprint: SPRINT_5100_0007_0005_connector_fixtures
|
||||
// Tasks: CONN-FIX-012, CONN-FIX-013
|
||||
// Description: Base class for connector security tests including URL allowlist,
|
||||
// redirect handling, max payload size, and decompression bomb protection.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TestKit.Connectors;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for connector security tests.
|
||||
/// Tests URL allowlist, redirect handling, max payload size, and decompression bombs.
|
||||
/// </summary>
|
||||
public abstract class ConnectorSecurityTestBase : IDisposable
|
||||
{
|
||||
protected readonly ConnectorHttpFixture HttpFixture;
|
||||
private bool _disposed;
|
||||
|
||||
protected ConnectorSecurityTestBase()
|
||||
{
|
||||
HttpFixture = new ConnectorHttpFixture();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the connector name for logging purposes.
|
||||
/// </summary>
|
||||
protected abstract string ConnectorName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the list of allowed URL patterns/domains for this connector.
|
||||
/// </summary>
|
||||
protected abstract IReadOnlyList<string> AllowedUrlPatterns { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the maximum allowed payload size in bytes.
|
||||
/// Default is 50MB.
|
||||
/// </summary>
|
||||
protected virtual long MaxPayloadSizeBytes => 50 * 1024 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the maximum allowed decompression ratio.
|
||||
/// Default is 100:1 (100 bytes uncompressed per 1 byte compressed).
|
||||
/// </summary>
|
||||
protected virtual int MaxDecompressionRatio => 100;
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to fetch from URL and returns whether it was allowed.
|
||||
/// Should implement the connector's actual URL validation logic.
|
||||
/// </summary>
|
||||
protected abstract Task<(bool Allowed, string? ErrorMessage)> TryFetchUrlAsync(
|
||||
string url,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the connector rejects URLs not in the allowlist.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public virtual async Task NonAllowlistedUrl_IsRejected()
|
||||
{
|
||||
var disallowedUrls = new[]
|
||||
{
|
||||
"https://evil.example.com/api/data",
|
||||
"http://malicious.test/feed",
|
||||
"https://attacker.io/redirect",
|
||||
"ftp://files.example.com/data.json",
|
||||
"file:///etc/passwd",
|
||||
"data:text/html,<script>alert(1)</script>",
|
||||
"javascript:alert(1)"
|
||||
};
|
||||
|
||||
foreach (var url in disallowedUrls)
|
||||
{
|
||||
var (allowed, errorMessage) = await TryFetchUrlAsync(url);
|
||||
|
||||
allowed.Should().BeFalse(
|
||||
$"URL '{url}' should be rejected as it's not in the allowlist for {ConnectorName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that HTTPS downgrade to HTTP is rejected.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public virtual async Task HttpsToHttpDowngrade_IsRejected()
|
||||
{
|
||||
// Setup a redirect from HTTPS to HTTP
|
||||
var secureUrl = "https://secure.example.com/api";
|
||||
var insecureUrl = "http://secure.example.com/api";
|
||||
|
||||
// The connector should reject HTTP URLs or prevent HTTPS->HTTP redirects
|
||||
var (allowed, _) = await TryFetchUrlAsync(insecureUrl);
|
||||
|
||||
allowed.Should().BeFalse(
|
||||
$"HTTP URLs should be rejected for {ConnectorName} (HTTPS required)");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that localhost/internal URLs are rejected.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData("http://localhost/api")]
|
||||
[InlineData("http://127.0.0.1/api")]
|
||||
[InlineData("http://[::1]/api")]
|
||||
[InlineData("http://169.254.169.254/latest/meta-data")] // AWS metadata
|
||||
[InlineData("http://metadata.google.internal/computeMetadata/v1/")] // GCP metadata
|
||||
public virtual async Task InternalUrl_IsRejected(string internalUrl)
|
||||
{
|
||||
var (allowed, _) = await TryFetchUrlAsync(internalUrl);
|
||||
|
||||
allowed.Should().BeFalse(
|
||||
$"Internal URL '{internalUrl}' should be rejected for {ConnectorName}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that redirect chains are limited.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public virtual void MaxRedirectChain_IsEnforced()
|
||||
{
|
||||
// This test documents that redirect chains should be limited
|
||||
// Implementation varies by connector/HTTP client configuration
|
||||
|
||||
// The connector's HTTP client should have MaxAutomaticRedirections set
|
||||
// to a reasonable value (typically 5-10)
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a gzip bomb payload (small compressed, huge uncompressed).
|
||||
/// </summary>
|
||||
protected static byte[] CreateGzipBomb(int uncompressedSize, int repetitions = 1)
|
||||
{
|
||||
// Create a highly compressible payload
|
||||
var pattern = new byte[1024];
|
||||
Array.Fill(pattern, (byte)'A');
|
||||
|
||||
using var output = new MemoryStream();
|
||||
using (var gzip = new GZipStream(output, CompressionLevel.Optimal))
|
||||
{
|
||||
for (int r = 0; r < repetitions; r++)
|
||||
{
|
||||
for (int i = 0; i < uncompressedSize / pattern.Length; i++)
|
||||
{
|
||||
gzip.Write(pattern, 0, pattern.Length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a nested gzip bomb (gzip within gzip).
|
||||
/// </summary>
|
||||
protected static byte[] CreateNestedGzipBomb(int depth, int baseSize)
|
||||
{
|
||||
var data = Encoding.UTF8.GetBytes(new string('A', baseSize));
|
||||
|
||||
for (int i = 0; i < depth; i++)
|
||||
{
|
||||
using var output = new MemoryStream();
|
||||
using (var gzip = new GZipStream(output, CompressionLevel.Optimal))
|
||||
{
|
||||
gzip.Write(data, 0, data.Length);
|
||||
}
|
||||
data = output.ToArray();
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that a URL matches the allowlist patterns.
|
||||
/// </summary>
|
||||
protected bool IsUrlInAllowlist(string url)
|
||||
{
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (var pattern in AllowedUrlPatterns)
|
||||
{
|
||||
if (MatchesPattern(url, pattern) || MatchesPattern(uri.Host, pattern))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool MatchesPattern(string value, string pattern)
|
||||
{
|
||||
if (pattern == "*") return true;
|
||||
if (value == pattern) return true;
|
||||
|
||||
if (pattern.StartsWith("*."))
|
||||
{
|
||||
var suffix = pattern[1..]; // e.g., ".github.com"
|
||||
return value.EndsWith(suffix, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (pattern.EndsWith("*"))
|
||||
{
|
||||
var prefix = pattern[..^1];
|
||||
return value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
if (disposing)
|
||||
{
|
||||
HttpFixture.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides common security test data for connectors.
|
||||
/// </summary>
|
||||
public static class ConnectorSecurityTestData
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets a list of URLs that should be rejected by all connectors.
|
||||
/// </summary>
|
||||
public static TheoryData<string> GetMaliciousUrls() => new()
|
||||
{
|
||||
"file:///etc/passwd",
|
||||
"file:///C:/Windows/System32/config/SAM",
|
||||
"data:text/html,<script>alert(1)</script>",
|
||||
"javascript:alert(1)",
|
||||
"http://localhost/api",
|
||||
"http://127.0.0.1/api",
|
||||
"http://[::1]/api",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://metadata.google.internal/",
|
||||
"http://192.168.1.1/admin",
|
||||
"http://10.0.0.1/internal",
|
||||
"gopher://evil.com:70/",
|
||||
"dict://evil.com:2628/"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Gets a list of SSRF bypass attempts.
|
||||
/// </summary>
|
||||
public static TheoryData<string> GetSsrfBypassAttempts() => new()
|
||||
{
|
||||
"http://2130706433/", // 127.0.0.1 as decimal
|
||||
"http://0x7f000001/", // 127.0.0.1 as hex
|
||||
"http://017700000001/", // 127.0.0.1 as octal
|
||||
"http://127.1/",
|
||||
"http://127.0.1/",
|
||||
"http://0/",
|
||||
"http://[0:0:0:0:0:ffff:127.0.0.1]/",
|
||||
"http://localtest.me/", // DNS rebinding
|
||||
"http://customer1.app.localhost.my.company.127.0.0.1.nip.io/"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Gets oversized payload test data.
|
||||
/// </summary>
|
||||
public static byte[] CreateOversizedPayload(long sizeBytes)
|
||||
{
|
||||
var payload = new byte[sizeBytes];
|
||||
Array.Fill(payload, (byte)'{');
|
||||
return payload;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// FlakyToDeterministicPattern.cs
|
||||
// Sprint: SPRINT_5100_0007_0001 (Testing Strategy)
|
||||
// Task: TEST-STRAT-5100-006 - Convert flaky E2E tests to deterministic integration tests
|
||||
// Description: Template demonstrating how to convert common flaky test patterns into
|
||||
// deterministic integration tests. Use as reference when refactoring tests.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace StellaOps.TestKit.Templates;
|
||||
|
||||
/// <summary>
|
||||
/// Template demonstrating the conversion of common flaky test patterns to deterministic tests.
|
||||
/// This class documents anti-patterns and their deterministic replacements.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Common sources of test flakiness and their solutions:
|
||||
///
|
||||
/// 1. **DateTime.Now/UtcNow** → Use injected TimeProvider or DeterministicTime
|
||||
/// 2. **Random without seed** → Use DeterministicRandom with fixed seed
|
||||
/// 3. **Task.Delay for timing** → Use polling with configurable timeout or fake timers
|
||||
/// 4. **External service calls** → Use HttpFixtureServer or mocks
|
||||
/// 5. **Ordering assumptions** → Ensure explicit ORDER BY or use sorted assertions
|
||||
/// 6. **Parallel test interference** → Use test isolation (schema-per-test, unique IDs)
|
||||
/// 7. **Environment dependencies** → Use TestContainers with fixed versions
|
||||
/// </remarks>
|
||||
public static class FlakyToDeterministicPattern
|
||||
{
|
||||
#region Pattern 1: Replace DateTime.Now with TimeProvider
|
||||
|
||||
// FLAKY: Uses system clock - different results on each run
|
||||
// public void Flaky_DateTimeNow()
|
||||
// {
|
||||
// var record = new AuditRecord { CreatedAt = DateTime.UtcNow };
|
||||
// Assert.True(record.CreatedAt.Hour == 12); // Fails at any other hour
|
||||
// }
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic version using injected TimeProvider.
|
||||
/// </summary>
|
||||
public static void Deterministic_TimeProvider(TimeProvider timeProvider)
|
||||
{
|
||||
// Use fixed time: DateTimeOffset.Parse("2025-01-15T12:00:00Z")
|
||||
var fixedTime = timeProvider.GetUtcNow();
|
||||
var record = new AuditRecordExample { CreatedAt = fixedTime };
|
||||
|
||||
// Assertions now pass regardless of actual system time
|
||||
if (record.CreatedAt.Hour != 12)
|
||||
{
|
||||
throw new InvalidOperationException("Time should be fixed at noon");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pattern 2: Replace Random with Seeded Random
|
||||
|
||||
// FLAKY: Different random sequence each run
|
||||
// public void Flaky_Random()
|
||||
// {
|
||||
// var random = new Random();
|
||||
// var value = random.Next(1, 100);
|
||||
// Assert.Equal(42, value); // Almost never passes
|
||||
// }
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic version using seeded random.
|
||||
/// </summary>
|
||||
public static int Deterministic_SeededRandom(int seed = 12345)
|
||||
{
|
||||
// Same seed always produces same sequence
|
||||
var random = new Random(seed);
|
||||
return random.Next(1, 100); // Always returns same value for same seed
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pattern 3: Replace Task.Delay with Polling
|
||||
|
||||
// FLAKY: Arbitrary delay may be too short or too long
|
||||
// public async Task Flaky_TaskDelay()
|
||||
// {
|
||||
// await service.StartAsync();
|
||||
// await Task.Delay(2000); // May be too short in CI
|
||||
// Assert.True(service.IsReady);
|
||||
// }
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic version using polling with timeout.
|
||||
/// </summary>
|
||||
public static async Task Deterministic_Polling(
|
||||
Func<bool> condition,
|
||||
TimeSpan timeout,
|
||||
TimeSpan? pollInterval = null)
|
||||
{
|
||||
var interval = pollInterval ?? TimeSpan.FromMilliseconds(100);
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
|
||||
while (stopwatch.Elapsed < timeout)
|
||||
{
|
||||
if (condition())
|
||||
{
|
||||
return;
|
||||
}
|
||||
await Task.Delay(interval);
|
||||
}
|
||||
|
||||
throw new TimeoutException($"Condition not met within {timeout.TotalSeconds}s");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Even better: Use fake timers in tests for instant execution.
|
||||
/// </summary>
|
||||
public static async Task Deterministic_FakeTimer(
|
||||
FakeTimeProvider fakeTime,
|
||||
Func<Task> actionThatWaits)
|
||||
{
|
||||
var task = actionThatWaits();
|
||||
|
||||
// Advance fake time instantly - no actual waiting
|
||||
fakeTime.Advance(TimeSpan.FromMinutes(5));
|
||||
|
||||
await task;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pattern 4: Replace External HTTP with Fixture Server
|
||||
|
||||
// FLAKY: Depends on external service availability
|
||||
// public async Task Flaky_ExternalHttp()
|
||||
// {
|
||||
// var client = new HttpClient();
|
||||
// var response = await client.GetAsync("https://api.example.com/data");
|
||||
// Assert.True(response.IsSuccessStatusCode);
|
||||
// }
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic version using HttpFixtureServer.
|
||||
/// </summary>
|
||||
public static async Task Deterministic_HttpFixture(HttpClient fixturedClient)
|
||||
{
|
||||
// HttpFixtureServer configured with:
|
||||
// server.Given("/data").Respond(HttpStatusCode.OK, fixedPayload);
|
||||
|
||||
var response = await fixturedClient.GetAsync("/data");
|
||||
|
||||
// Always succeeds because response is mocked
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
throw new InvalidOperationException("Fixtured response should succeed");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pattern 5: Ensure Deterministic Ordering
|
||||
|
||||
// FLAKY: Order not guaranteed without explicit ORDER BY
|
||||
// public void Flaky_Ordering()
|
||||
// {
|
||||
// var items = repository.GetAll();
|
||||
// Assert.Equal("first", items[0].Name); // May fail due to DB ordering
|
||||
// }
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic version with explicit ordering.
|
||||
/// </summary>
|
||||
public static void Deterministic_ExplicitOrdering<T>(
|
||||
IEnumerable<T> items,
|
||||
Func<T, object> orderBy)
|
||||
{
|
||||
// Always sort before assertions
|
||||
var sorted = items.OrderBy(orderBy).ToList();
|
||||
|
||||
// Now ordering is deterministic
|
||||
// Assert.Equal("first", sorted[0].Name);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Alternative: Use set-based assertions that ignore order.
|
||||
/// </summary>
|
||||
public static void Deterministic_SetAssertion<T>(
|
||||
IEnumerable<T> actual,
|
||||
IEnumerable<T> expected)
|
||||
{
|
||||
var actualSet = actual.ToHashSet();
|
||||
var expectedSet = expected.ToHashSet();
|
||||
|
||||
if (!actualSet.SetEquals(expectedSet))
|
||||
{
|
||||
throw new InvalidOperationException("Sets are not equal");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pattern 6: Test Isolation
|
||||
|
||||
// FLAKY: Parallel tests interfere with each other
|
||||
// public async Task Flaky_SharedState()
|
||||
// {
|
||||
// await db.InsertAsync(new Item { Id = 1, Name = "test" });
|
||||
// var item = await db.GetAsync(1);
|
||||
// Assert.Equal("test", item.Name); // Fails if another test uses Id=1
|
||||
// }
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic version with unique identifiers.
|
||||
/// </summary>
|
||||
public static string GenerateTestId(string testName)
|
||||
{
|
||||
// Each test gets unique ID based on test name + timestamp
|
||||
return $"{testName}-{Guid.NewGuid():N}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Alternative: Use schema-per-test isolation.
|
||||
/// </summary>
|
||||
public static string GetIsolatedSchema(string testName)
|
||||
{
|
||||
// PostgresFixture.SchemaPerTest mode creates:
|
||||
return $"test_{testName.Replace(".", "_").ToLowerInvariant()}";
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Pattern 7: Fixed Container Versions
|
||||
|
||||
// FLAKY: Different container versions may behave differently
|
||||
// public class Flaky_ContainerVersion
|
||||
// {
|
||||
// private readonly PostgresContainer _container = new PostgresContainer("postgres:latest");
|
||||
// }
|
||||
|
||||
/// <summary>
|
||||
/// Deterministic version with pinned versions.
|
||||
/// </summary>
|
||||
public static class DeterministicContainerVersions
|
||||
{
|
||||
// Pin all container versions for reproducibility
|
||||
public const string PostgresVersion = "postgres:16.1-alpine";
|
||||
public const string ValkeyVersion = "valkey/valkey:8.0.1";
|
||||
public const string RabbitMqVersion = "rabbitmq:3.12.10-management-alpine";
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Example Types
|
||||
|
||||
private class AuditRecordExample
|
||||
{
|
||||
public DateTimeOffset CreatedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fake TimeProvider for testing time-dependent code.
|
||||
/// </summary>
|
||||
public class FakeTimeProvider : TimeProvider
|
||||
{
|
||||
private DateTimeOffset _currentTime;
|
||||
|
||||
public FakeTimeProvider(DateTimeOffset startTime)
|
||||
{
|
||||
_currentTime = startTime;
|
||||
}
|
||||
|
||||
public override DateTimeOffset GetUtcNow() => _currentTime;
|
||||
|
||||
public void Advance(TimeSpan duration)
|
||||
{
|
||||
_currentTime = _currentTime.Add(duration);
|
||||
}
|
||||
|
||||
public void SetTime(DateTimeOffset newTime)
|
||||
{
|
||||
_currentTime = newTime;
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user