522 lines
18 KiB
C#
522 lines
18 KiB
C#
using FluentAssertions;
|
|
using StellaOps.Policy.Engine.DeterminismGuard;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Policy.Engine.Tests.DeterminismGuard;
|
|
|
|
/// <summary>
|
|
/// Deep verification tests for determinism guards covering pattern detection gaps,
|
|
/// ValidateContext, FailOnSeverity threshold, GuardedPolicyEvaluatorBuilder,
|
|
/// floating-point/unstable-iteration warnings, socket detection, and scope lifecycle.
|
|
/// </summary>
|
|
public sealed class DeterminismGuardDeepTests
|
|
{
|
|
#region Additional Pattern Detection
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsDateTimeOffsetNow()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "var now = DateTimeOffset.Now;";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.ViolationType == "DateTimeOffset.Now" &&
|
|
v.Category == DeterminismViolationCategory.WallClock);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsDateTimeOffsetUtcNow()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "var now = DateTimeOffset.UtcNow;";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.ViolationType == "DateTimeOffset.UtcNow" &&
|
|
v.Category == DeterminismViolationCategory.WallClock);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsCryptoRandom()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "var bytes = RandomNumberGenerator.GetBytes(32);";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.ViolationType == "RandomNumberGenerator" &&
|
|
v.Category == DeterminismViolationCategory.RandomNumber);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsSocketClasses()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = """
|
|
var tcp = new TcpClient("localhost", 80);
|
|
var udp = new UdpClient(9090);
|
|
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
|
""";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().HaveCount(3);
|
|
result.Violations.Should().OnlyContain(v =>
|
|
v.Category == DeterminismViolationCategory.NetworkAccess &&
|
|
v.Severity == DeterminismViolationSeverity.Critical);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsWebClient()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "using var client = new WebClient();";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.ViolationType == "WebClient" &&
|
|
v.Category == DeterminismViolationCategory.NetworkAccess);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsEnvironmentMachineName()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "var name = Environment.MachineName;";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.ViolationType == "Environment.MachineName" &&
|
|
v.Category == DeterminismViolationCategory.EnvironmentAccess &&
|
|
v.Severity == DeterminismViolationSeverity.Warning);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsFloatingPointComparison()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "double score == 7.5;";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.Category == DeterminismViolationCategory.FloatingPointHazard &&
|
|
v.Severity == DeterminismViolationSeverity.Warning);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsDictionaryIteration()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "foreach (var item in myDictionary)";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.Category == DeterminismViolationCategory.UnstableIteration);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_DetectsHashSetIteration()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "foreach (var item in myHashSet)";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.Category == DeterminismViolationCategory.UnstableIteration);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_MultipleViolationCategories_ReportsAll()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = """
|
|
var now = DateTime.Now;
|
|
var rng = new Random();
|
|
var id = Guid.NewGuid();
|
|
private readonly HttpClient _client = new();
|
|
""";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().HaveCountGreaterThanOrEqualTo(4);
|
|
result.Violations.Select(v => v.Category).Distinct()
|
|
.Should().Contain(DeterminismViolationCategory.WallClock)
|
|
.And.Contain(DeterminismViolationCategory.RandomNumber)
|
|
.And.Contain(DeterminismViolationCategory.GuidGeneration)
|
|
.And.Contain(DeterminismViolationCategory.NetworkAccess);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region ValidateContext Tests
|
|
|
|
[Fact]
|
|
public void ValidateContext_NullContext_DetectsViolation()
|
|
{
|
|
var guard = new DeterminismGuardService();
|
|
|
|
var result = guard.ValidateContext<object>(null!, "TestContext");
|
|
|
|
result.Passed.Should().BeFalse();
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.Category == DeterminismViolationCategory.Other &&
|
|
v.ViolationType == "NullContext" &&
|
|
v.Message.Contains("TestContext"));
|
|
}
|
|
|
|
[Fact]
|
|
public void ValidateContext_ValidContext_Passes()
|
|
{
|
|
var guard = new DeterminismGuardService();
|
|
|
|
var result = guard.ValidateContext(new { Score = 7.5 }, "ScoringContext");
|
|
|
|
result.Passed.Should().BeTrue();
|
|
result.Violations.Should().BeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void ValidateContext_EnforcementDisabled_NullContextPassesButReportsViolation()
|
|
{
|
|
var options = new DeterminismGuardOptions { EnforcementEnabled = false };
|
|
var guard = new DeterminismGuardService(options);
|
|
|
|
var result = guard.ValidateContext<object>(null!, "TestContext");
|
|
|
|
result.Passed.Should().BeTrue(); // Enforcement disabled = always passes
|
|
result.Violations.Should().NotBeEmpty(); // But still reports violations
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region FailOnSeverity Threshold Tests
|
|
|
|
[Fact]
|
|
public void FailOnSeverity_Error_WarningViolationsDoNotCauseFailure()
|
|
{
|
|
var options = new DeterminismGuardOptions
|
|
{
|
|
EnforcementEnabled = true,
|
|
FailOnSeverity = DeterminismViolationSeverity.Error
|
|
};
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
// Environment.MachineName is a Warning-level violation
|
|
var source = "var name = Environment.MachineName;";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", options);
|
|
|
|
result.Passed.Should().BeTrue(); // Warning < Error threshold
|
|
result.Violations.Should().NotBeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void FailOnSeverity_Error_ErrorViolationsCauseFailure()
|
|
{
|
|
var options = new DeterminismGuardOptions
|
|
{
|
|
EnforcementEnabled = true,
|
|
FailOnSeverity = DeterminismViolationSeverity.Error
|
|
};
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "var now = DateTime.Now;";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", options);
|
|
|
|
result.Passed.Should().BeFalse(); // Error >= Error threshold
|
|
}
|
|
|
|
[Fact]
|
|
public void FailOnSeverity_Critical_ErrorViolationsDoNotCauseFailure()
|
|
{
|
|
var options = new DeterminismGuardOptions
|
|
{
|
|
EnforcementEnabled = true,
|
|
FailOnSeverity = DeterminismViolationSeverity.Critical
|
|
};
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
// DateTime.Now is Error severity
|
|
var source = "var now = DateTime.Now;";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", options);
|
|
|
|
result.Passed.Should().BeTrue(); // Error < Critical threshold
|
|
}
|
|
|
|
[Fact]
|
|
public void FailOnSeverity_Critical_CriticalViolationsCauseFailure()
|
|
{
|
|
var options = new DeterminismGuardOptions
|
|
{
|
|
EnforcementEnabled = true,
|
|
FailOnSeverity = DeterminismViolationSeverity.Critical
|
|
};
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
// HttpClient is Critical severity
|
|
var source = "private readonly HttpClient _client = new();";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", options);
|
|
|
|
result.Passed.Should().BeFalse(); // Critical >= Critical threshold
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region GuardedPolicyEvaluatorBuilder Tests
|
|
|
|
[Fact]
|
|
public void Builder_CreateDevelopment_HasNoEnforcement()
|
|
{
|
|
var evaluator = GuardedPolicyEvaluatorBuilder.CreateDevelopment();
|
|
|
|
// Development mode: no enforcement, so reporting a critical violation should not throw
|
|
var result = evaluator.Evaluate("dev-scope", DateTimeOffset.UtcNow, scope =>
|
|
{
|
|
scope.ReportViolation(new DeterminismViolation
|
|
{
|
|
Category = DeterminismViolationCategory.NetworkAccess,
|
|
ViolationType = "HttpClient",
|
|
Message = "Dev mode test",
|
|
Severity = DeterminismViolationSeverity.Critical
|
|
});
|
|
return "ok";
|
|
});
|
|
|
|
result.Succeeded.Should().BeTrue(); // Enforcement disabled in dev mode
|
|
result.Result.Should().Be("ok");
|
|
result.HasViolations.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_CreateProduction_HasEnforcement()
|
|
{
|
|
var evaluator = GuardedPolicyEvaluatorBuilder.CreateProduction();
|
|
|
|
var result = evaluator.Evaluate("prod-scope", DateTimeOffset.UtcNow, scope =>
|
|
{
|
|
scope.ReportViolation(new DeterminismViolation
|
|
{
|
|
Category = DeterminismViolationCategory.WallClock,
|
|
ViolationType = "DateTime.Now",
|
|
Message = "Wall clock in prod",
|
|
Severity = DeterminismViolationSeverity.Error
|
|
});
|
|
return "should not return";
|
|
});
|
|
|
|
result.Succeeded.Should().BeFalse();
|
|
result.WasBlocked.Should().BeTrue();
|
|
}
|
|
|
|
[Fact]
|
|
public void Builder_CustomConfiguration_AppliesCorrectly()
|
|
{
|
|
var evaluator = new GuardedPolicyEvaluatorBuilder()
|
|
.WithEnforcement(true)
|
|
.FailOnSeverity(DeterminismViolationSeverity.Critical)
|
|
.WithRuntimeMonitoring(true)
|
|
.ExcludePatterns("test_", "spec_")
|
|
.Build();
|
|
|
|
// Error-level violations should pass since FailOnSeverity is Critical
|
|
var result = evaluator.Evaluate("custom-scope", DateTimeOffset.UtcNow, scope =>
|
|
{
|
|
scope.ReportViolation(new DeterminismViolation
|
|
{
|
|
Category = DeterminismViolationCategory.WallClock,
|
|
ViolationType = "DateTime.Now",
|
|
Message = "Error-level warning",
|
|
Severity = DeterminismViolationSeverity.Error
|
|
});
|
|
return 42;
|
|
});
|
|
|
|
result.Succeeded.Should().BeTrue();
|
|
result.Result.Should().Be(42);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Scope Lifecycle Tests
|
|
|
|
[Fact]
|
|
public void Scope_Complete_CountsBySeverity()
|
|
{
|
|
var guard = new DeterminismGuardService(DeterminismGuardOptions.Development);
|
|
using var scope = guard.CreateScope("lifecycle-test", DateTimeOffset.UtcNow);
|
|
|
|
scope.ReportViolation(new DeterminismViolation
|
|
{
|
|
Category = DeterminismViolationCategory.WallClock,
|
|
ViolationType = "Test1",
|
|
Message = "Warning 1",
|
|
Severity = DeterminismViolationSeverity.Warning
|
|
});
|
|
scope.ReportViolation(new DeterminismViolation
|
|
{
|
|
Category = DeterminismViolationCategory.RandomNumber,
|
|
ViolationType = "Test2",
|
|
Message = "Warning 2",
|
|
Severity = DeterminismViolationSeverity.Warning
|
|
});
|
|
scope.ReportViolation(new DeterminismViolation
|
|
{
|
|
Category = DeterminismViolationCategory.NetworkAccess,
|
|
ViolationType = "Test3",
|
|
Message = "Error 1",
|
|
Severity = DeterminismViolationSeverity.Error
|
|
});
|
|
|
|
var result = scope.Complete();
|
|
|
|
result.Violations.Should().HaveCount(3);
|
|
result.CountBySeverity[DeterminismViolationSeverity.Warning].Should().Be(2);
|
|
result.CountBySeverity[DeterminismViolationSeverity.Error].Should().Be(1);
|
|
result.AnalysisDurationMs.Should().BeGreaterThanOrEqualTo(0);
|
|
}
|
|
|
|
[Fact]
|
|
public void Scope_ScopeId_IsPreserved()
|
|
{
|
|
var guard = new DeterminismGuardService();
|
|
using var scope = guard.CreateScope("my-scope-id", DateTimeOffset.UtcNow);
|
|
|
|
scope.ScopeId.Should().Be("my-scope-id");
|
|
}
|
|
|
|
[Fact]
|
|
public void Scope_NullScopeId_ThrowsArgumentNullException()
|
|
{
|
|
var guard = new DeterminismGuardService();
|
|
|
|
FluentActions.Invoking(() => guard.CreateScope(null!, DateTimeOffset.UtcNow))
|
|
.Should().Throw<ArgumentNullException>();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region DeterministicTimeProvider Tests
|
|
|
|
[Fact]
|
|
public void DeterministicTimeProvider_MultipleCallsReturnSameValue()
|
|
{
|
|
var fixedTime = new DateTimeOffset(2026, 2, 12, 10, 0, 0, TimeSpan.Zero);
|
|
var provider = new DeterministicTimeProvider(fixedTime);
|
|
|
|
// 100 calls should all return the same value
|
|
for (int i = 0; i < 100; i++)
|
|
{
|
|
provider.GetUtcNow().Should().Be(fixedTime);
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region GuardedEvaluationResult Properties
|
|
|
|
[Fact]
|
|
public void GuardedEvaluationResult_ViolationCountBySeverity_Works()
|
|
{
|
|
var evaluator = new GuardedPolicyEvaluator(DeterminismGuardOptions.Development);
|
|
|
|
var result = evaluator.Evaluate("count-test", DateTimeOffset.UtcNow, scope =>
|
|
{
|
|
scope.ReportViolation(new DeterminismViolation
|
|
{
|
|
Category = DeterminismViolationCategory.WallClock,
|
|
ViolationType = "T1",
|
|
Message = "W1",
|
|
Severity = DeterminismViolationSeverity.Warning
|
|
});
|
|
scope.ReportViolation(new DeterminismViolation
|
|
{
|
|
Category = DeterminismViolationCategory.WallClock,
|
|
ViolationType = "T2",
|
|
Message = "E1",
|
|
Severity = DeterminismViolationSeverity.Error
|
|
});
|
|
return "done";
|
|
});
|
|
|
|
result.ViolationCountBySeverity.Should().ContainKey(DeterminismViolationSeverity.Warning);
|
|
result.ViolationCountBySeverity[DeterminismViolationSeverity.Warning].Should().Be(1);
|
|
result.ViolationCountBySeverity[DeterminismViolationSeverity.Error].Should().Be(1);
|
|
result.HasViolations.Should().BeTrue();
|
|
result.WasBlocked.Should().BeFalse();
|
|
result.ScopeId.Should().Be("count-test");
|
|
}
|
|
|
|
[Fact]
|
|
public void Evaluate_UnexpectedException_RecordsAsCriticalViolation()
|
|
{
|
|
var evaluator = new GuardedPolicyEvaluator();
|
|
|
|
var result = evaluator.Evaluate<string>("exception-test", DateTimeOffset.UtcNow, scope =>
|
|
{
|
|
throw new InvalidOperationException("Test exception");
|
|
});
|
|
|
|
result.Succeeded.Should().BeFalse();
|
|
result.Exception.Should().NotBeNull();
|
|
result.Exception.Should().BeOfType<InvalidOperationException>();
|
|
result.BlockingViolation.Should().NotBeNull();
|
|
result.BlockingViolation!.ViolationType.Should().Be("EvaluationException");
|
|
result.BlockingViolation.Severity.Should().Be(DeterminismViolationSeverity.Critical);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region DeterminismAnalysisResult.Pass Factory
|
|
|
|
[Fact]
|
|
public void DeterminismAnalysisResult_Pass_CreatesCleanResult()
|
|
{
|
|
var result = DeterminismAnalysisResult.Pass(42, true);
|
|
|
|
result.Passed.Should().BeTrue();
|
|
result.Violations.Should().BeEmpty();
|
|
result.CountBySeverity.Should().BeEmpty();
|
|
result.AnalysisDurationMs.Should().Be(42);
|
|
result.EnforcementEnabled.Should().BeTrue();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Violation Remediation Messages
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_ViolationsIncludeRemediation()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "var now = DateTime.Now;";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle()
|
|
.Which.Remediation.Should().NotBeNullOrWhiteSpace();
|
|
}
|
|
|
|
[Fact]
|
|
public void AnalyzeSource_FileReadViolation_HasCriticalSeverity()
|
|
{
|
|
var analyzer = new ProhibitedPatternAnalyzer();
|
|
var source = "var text = File.ReadAllText(\"config.json\");";
|
|
|
|
var result = analyzer.AnalyzeSource(source, "test.cs", DeterminismGuardOptions.Default);
|
|
|
|
result.Violations.Should().ContainSingle(v =>
|
|
v.Category == DeterminismViolationCategory.FileSystemAccess &&
|
|
v.Severity == DeterminismViolationSeverity.Critical);
|
|
}
|
|
|
|
#endregion
|
|
}
|