Add determinism tests for verdict artifact generation and update SHA256 sums script
- Implemented comprehensive tests for verdict artifact generation to ensure deterministic outputs across various scenarios, including identical inputs, parallel execution, and change ordering. - Created helper methods for generating sample verdict inputs and computing canonical hashes. - Added tests to validate the stability of canonical hashes, proof spine ordering, and summary statistics. - Introduced a new PowerShell script to update SHA256 sums for files, ensuring accurate hash generation and file integrity checks.
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,191 @@
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TestKit.Connectors;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for connector live schema drift detection tests.
|
||||
/// These tests fetch from live upstream sources and compare against stored fixtures.
|
||||
///
|
||||
/// IMPORTANT: These tests are opt-in and disabled by default.
|
||||
/// To enable: set STELLAOPS_LIVE_TESTS=true environment variable.
|
||||
/// To also auto-update fixtures on drift: set STELLAOPS_UPDATE_FIXTURES=true.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Usage:
|
||||
/// <code>
|
||||
/// [Trait("Category", TestCategories.Live)]
|
||||
/// public class NvdLiveSchemaTests : ConnectorLiveSchemaTestBase
|
||||
/// {
|
||||
/// protected override string FixturesDirectory => "Fixtures";
|
||||
/// protected override string ConnectorName => "NVD";
|
||||
///
|
||||
/// protected override IEnumerable<LiveSchemaTestCase> GetTestCases()
|
||||
/// {
|
||||
/// yield return new("typical-cve.json", "https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=CVE-2024-0001");
|
||||
/// }
|
||||
///
|
||||
/// [Fact]
|
||||
/// public Task DetectSchemaDrift() => RunSchemaDriftTestsAsync();
|
||||
/// }
|
||||
/// </code>
|
||||
/// </remarks>
|
||||
public abstract class ConnectorLiveSchemaTestBase : IAsyncLifetime
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly FixtureUpdater _fixtureUpdater;
|
||||
private readonly List<FixtureDriftReport> _driftReports = new();
|
||||
|
||||
protected ConnectorLiveSchemaTestBase()
|
||||
{
|
||||
_httpClient = new HttpClient
|
||||
{
|
||||
Timeout = TimeSpan.FromSeconds(30)
|
||||
};
|
||||
_fixtureUpdater = new FixtureUpdater(FixturesDirectory, _httpClient);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the base directory for test fixtures (relative to test assembly).
|
||||
/// </summary>
|
||||
protected abstract string FixturesDirectory { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the connector name for logging purposes.
|
||||
/// </summary>
|
||||
protected abstract string ConnectorName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the test cases mapping fixtures to live URLs.
|
||||
/// </summary>
|
||||
protected abstract IEnumerable<LiveSchemaTestCase> GetTestCases();
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if live tests are enabled.
|
||||
/// </summary>
|
||||
protected static bool IsEnabled =>
|
||||
Environment.GetEnvironmentVariable("STELLAOPS_LIVE_TESTS") == "true";
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if fixture auto-update is enabled.
|
||||
/// </summary>
|
||||
protected static bool IsAutoUpdateEnabled =>
|
||||
Environment.GetEnvironmentVariable("STELLAOPS_UPDATE_FIXTURES") == "true";
|
||||
|
||||
/// <summary>
|
||||
/// Optional request headers for live requests.
|
||||
/// </summary>
|
||||
protected virtual Dictionary<string, string> RequestHeaders => new();
|
||||
|
||||
/// <summary>
|
||||
/// Runs all schema drift tests for this connector.
|
||||
/// </summary>
|
||||
protected async Task RunSchemaDriftTestsAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
// Skip test when not explicitly enabled
|
||||
return;
|
||||
}
|
||||
|
||||
var testCases = GetTestCases().ToList();
|
||||
_driftReports.Clear();
|
||||
|
||||
foreach (var testCase in testCases)
|
||||
{
|
||||
var report = await _fixtureUpdater.CheckDriftAsync(testCase.LiveUrl, testCase.FixtureName, ct);
|
||||
_driftReports.Add(report);
|
||||
|
||||
if (report.HasDrift && IsAutoUpdateEnabled && !string.IsNullOrEmpty(report.LiveContent))
|
||||
{
|
||||
await _fixtureUpdater.UpdateJsonFixtureFromUrlAsync(testCase.LiveUrl, testCase.FixtureName, ct);
|
||||
}
|
||||
}
|
||||
|
||||
// Report all drift findings
|
||||
var driftCount = _driftReports.Count(r => r.HasDrift);
|
||||
if (driftCount > 0)
|
||||
{
|
||||
var driftMessages = _driftReports
|
||||
.Where(r => r.HasDrift)
|
||||
.Select(r => $" - {r.FixtureName}: {r.Message}");
|
||||
|
||||
var message = $"Schema drift detected in {driftCount}/{_driftReports.Count} fixtures for {ConnectorName}:\n" +
|
||||
string.Join("\n", driftMessages);
|
||||
|
||||
if (IsAutoUpdateEnabled)
|
||||
{
|
||||
// Warn but don't fail when auto-updating
|
||||
Console.WriteLine($"[WARN] {message}");
|
||||
Console.WriteLine($"[INFO] Fixtures have been auto-updated. Review changes before committing.");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Fail test when drift detected without auto-update
|
||||
Assert.Fail(message + "\n\nSet STELLAOPS_UPDATE_FIXTURES=true to auto-update fixtures.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the drift reports from the last test run.
|
||||
/// </summary>
|
||||
public IReadOnlyList<FixtureDriftReport> DriftReports => _driftReports.AsReadOnly();
|
||||
|
||||
public Task InitializeAsync()
|
||||
{
|
||||
foreach (var (key, value) in RequestHeaders)
|
||||
{
|
||||
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation(key, value);
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task DisposeAsync()
|
||||
{
|
||||
_httpClient.Dispose();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test case for live schema drift detection.
|
||||
/// </summary>
|
||||
/// <param name="FixtureName">The fixture file name in the fixtures directory.</param>
|
||||
/// <param name="LiveUrl">The live URL to fetch for comparison.</param>
|
||||
/// <param name="Description">Optional description of what this fixture tests.</param>
|
||||
public sealed record LiveSchemaTestCase(
|
||||
string FixtureName,
|
||||
string LiveUrl,
|
||||
string? Description = null);
|
||||
|
||||
/// <summary>
|
||||
/// Attribute to mark tests that require live external services.
|
||||
/// These tests are skipped unless STELLAOPS_LIVE_TESTS=true.
|
||||
/// </summary>
|
||||
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false)]
|
||||
public sealed class LiveTestAttribute : FactAttribute
|
||||
{
|
||||
public LiveTestAttribute()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("STELLAOPS_LIVE_TESTS") != "true")
|
||||
{
|
||||
Skip = "Live tests are disabled. Set STELLAOPS_LIVE_TESTS=true to enable.";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Theory attribute for live tests that are skipped unless explicitly enabled.
|
||||
/// </summary>
|
||||
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
|
||||
public sealed class LiveTheoryAttribute : TheoryAttribute
|
||||
{
|
||||
public LiveTheoryAttribute()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("STELLAOPS_LIVE_TESTS") != "true")
|
||||
{
|
||||
Skip = "Live tests are disabled. Set STELLAOPS_LIVE_TESTS=true to enable.";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// ConnectorSecurityTestBase.cs
|
||||
// Sprint: SPRINT_5100_0007_0005_connector_fixtures
|
||||
// Tasks: CONN-FIX-012, CONN-FIX-013
|
||||
// Description: Base class for connector security tests including URL allowlist,
|
||||
// redirect handling, max payload size, and decompression bomb protection.
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.IO.Compression;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.TestKit.Connectors;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for connector security tests.
|
||||
/// Tests URL allowlist, redirect handling, max payload size, and decompression bombs.
|
||||
/// </summary>
|
||||
public abstract class ConnectorSecurityTestBase : IDisposable
|
||||
{
|
||||
protected readonly ConnectorHttpFixture HttpFixture;
|
||||
private bool _disposed;
|
||||
|
||||
protected ConnectorSecurityTestBase()
|
||||
{
|
||||
HttpFixture = new ConnectorHttpFixture();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the connector name for logging purposes.
|
||||
/// </summary>
|
||||
protected abstract string ConnectorName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the list of allowed URL patterns/domains for this connector.
|
||||
/// </summary>
|
||||
protected abstract IReadOnlyList<string> AllowedUrlPatterns { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the maximum allowed payload size in bytes.
|
||||
/// Default is 50MB.
|
||||
/// </summary>
|
||||
protected virtual long MaxPayloadSizeBytes => 50 * 1024 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the maximum allowed decompression ratio.
|
||||
/// Default is 100:1 (100 bytes uncompressed per 1 byte compressed).
|
||||
/// </summary>
|
||||
protected virtual int MaxDecompressionRatio => 100;
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to fetch from URL and returns whether it was allowed.
|
||||
/// Should implement the connector's actual URL validation logic.
|
||||
/// </summary>
|
||||
protected abstract Task<(bool Allowed, string? ErrorMessage)> TryFetchUrlAsync(
|
||||
string url,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the connector rejects URLs not in the allowlist.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public virtual async Task NonAllowlistedUrl_IsRejected()
|
||||
{
|
||||
var disallowedUrls = new[]
|
||||
{
|
||||
"https://evil.example.com/api/data",
|
||||
"http://malicious.test/feed",
|
||||
"https://attacker.io/redirect",
|
||||
"ftp://files.example.com/data.json",
|
||||
"file:///etc/passwd",
|
||||
"data:text/html,<script>alert(1)</script>",
|
||||
"javascript:alert(1)"
|
||||
};
|
||||
|
||||
foreach (var url in disallowedUrls)
|
||||
{
|
||||
var (allowed, errorMessage) = await TryFetchUrlAsync(url);
|
||||
|
||||
allowed.Should().BeFalse(
|
||||
$"URL '{url}' should be rejected as it's not in the allowlist for {ConnectorName}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that HTTPS downgrade to HTTP is rejected.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public virtual async Task HttpsToHttpDowngrade_IsRejected()
|
||||
{
|
||||
// Setup a redirect from HTTPS to HTTP
|
||||
var secureUrl = "https://secure.example.com/api";
|
||||
var insecureUrl = "http://secure.example.com/api";
|
||||
|
||||
// The connector should reject HTTP URLs or prevent HTTPS->HTTP redirects
|
||||
var (allowed, _) = await TryFetchUrlAsync(insecureUrl);
|
||||
|
||||
allowed.Should().BeFalse(
|
||||
$"HTTP URLs should be rejected for {ConnectorName} (HTTPS required)");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that localhost/internal URLs are rejected.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData("http://localhost/api")]
|
||||
[InlineData("http://127.0.0.1/api")]
|
||||
[InlineData("http://[::1]/api")]
|
||||
[InlineData("http://169.254.169.254/latest/meta-data")] // AWS metadata
|
||||
[InlineData("http://metadata.google.internal/computeMetadata/v1/")] // GCP metadata
|
||||
public virtual async Task InternalUrl_IsRejected(string internalUrl)
|
||||
{
|
||||
var (allowed, _) = await TryFetchUrlAsync(internalUrl);
|
||||
|
||||
allowed.Should().BeFalse(
|
||||
$"Internal URL '{internalUrl}' should be rejected for {ConnectorName}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that redirect chains are limited.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public virtual void MaxRedirectChain_IsEnforced()
|
||||
{
|
||||
// This test documents that redirect chains should be limited
|
||||
// Implementation varies by connector/HTTP client configuration
|
||||
|
||||
// The connector's HTTP client should have MaxAutomaticRedirections set
|
||||
// to a reasonable value (typically 5-10)
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a gzip bomb payload (small compressed, huge uncompressed).
|
||||
/// </summary>
|
||||
protected static byte[] CreateGzipBomb(int uncompressedSize, int repetitions = 1)
|
||||
{
|
||||
// Create a highly compressible payload
|
||||
var pattern = new byte[1024];
|
||||
Array.Fill(pattern, (byte)'A');
|
||||
|
||||
using var output = new MemoryStream();
|
||||
using (var gzip = new GZipStream(output, CompressionLevel.Optimal))
|
||||
{
|
||||
for (int r = 0; r < repetitions; r++)
|
||||
{
|
||||
for (int i = 0; i < uncompressedSize / pattern.Length; i++)
|
||||
{
|
||||
gzip.Write(pattern, 0, pattern.Length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a nested gzip bomb (gzip within gzip).
|
||||
/// </summary>
|
||||
protected static byte[] CreateNestedGzipBomb(int depth, int baseSize)
|
||||
{
|
||||
var data = Encoding.UTF8.GetBytes(new string('A', baseSize));
|
||||
|
||||
for (int i = 0; i < depth; i++)
|
||||
{
|
||||
using var output = new MemoryStream();
|
||||
using (var gzip = new GZipStream(output, CompressionLevel.Optimal))
|
||||
{
|
||||
gzip.Write(data, 0, data.Length);
|
||||
}
|
||||
data = output.ToArray();
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates that a URL matches the allowlist patterns.
|
||||
/// </summary>
|
||||
protected bool IsUrlInAllowlist(string url)
|
||||
{
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (var pattern in AllowedUrlPatterns)
|
||||
{
|
||||
if (MatchesPattern(url, pattern) || MatchesPattern(uri.Host, pattern))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool MatchesPattern(string value, string pattern)
|
||||
{
|
||||
if (pattern == "*") return true;
|
||||
if (value == pattern) return true;
|
||||
|
||||
if (pattern.StartsWith("*."))
|
||||
{
|
||||
var suffix = pattern[1..]; // e.g., ".github.com"
|
||||
return value.EndsWith(suffix, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (pattern.EndsWith("*"))
|
||||
{
|
||||
var prefix = pattern[..^1];
|
||||
return value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
if (disposing)
|
||||
{
|
||||
HttpFixture.Dispose();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Provides common security test data for connectors.
|
||||
/// </summary>
|
||||
public static class ConnectorSecurityTestData
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets a list of URLs that should be rejected by all connectors.
|
||||
/// </summary>
|
||||
public static TheoryData<string> GetMaliciousUrls() => new()
|
||||
{
|
||||
"file:///etc/passwd",
|
||||
"file:///C:/Windows/System32/config/SAM",
|
||||
"data:text/html,<script>alert(1)</script>",
|
||||
"javascript:alert(1)",
|
||||
"http://localhost/api",
|
||||
"http://127.0.0.1/api",
|
||||
"http://[::1]/api",
|
||||
"http://169.254.169.254/latest/meta-data",
|
||||
"http://metadata.google.internal/",
|
||||
"http://192.168.1.1/admin",
|
||||
"http://10.0.0.1/internal",
|
||||
"gopher://evil.com:70/",
|
||||
"dict://evil.com:2628/"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Gets a list of SSRF bypass attempts.
|
||||
/// </summary>
|
||||
public static TheoryData<string> GetSsrfBypassAttempts() => new()
|
||||
{
|
||||
"http://2130706433/", // 127.0.0.1 as decimal
|
||||
"http://0x7f000001/", // 127.0.0.1 as hex
|
||||
"http://017700000001/", // 127.0.0.1 as octal
|
||||
"http://127.1/",
|
||||
"http://127.0.1/",
|
||||
"http://0/",
|
||||
"http://[0:0:0:0:0:ffff:127.0.0.1]/",
|
||||
"http://localtest.me/", // DNS rebinding
|
||||
"http://customer1.app.localhost.my.company.127.0.0.1.nip.io/"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Gets oversized payload test data.
|
||||
/// </summary>
|
||||
public static byte[] CreateOversizedPayload(long sizeBytes)
|
||||
{
|
||||
var payload = new byte[sizeBytes];
|
||||
Array.Fill(payload, (byte)'{');
|
||||
return payload;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user