490 lines
17 KiB
C#
490 lines
17 KiB
C#
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
// Copyright © 2025 StellaOps
|
|
// Sprint: SPRINT_8200_0012_0003_policy_engine_integration
|
|
// Task: PINT-8200-042 - Concurrent evaluation test: thread-safe EWS in policy pipeline
|
|
|
|
using FluentAssertions;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using StellaOps.Policy.Engine.Scoring.EvidenceWeightedScore;
|
|
using StellaOps.Signals.EvidenceWeightedScore;
|
|
using StellaOps.Signals.EvidenceWeightedScore.Normalizers;
|
|
using System.Collections.Concurrent;
|
|
using Xunit;
|
|
|
|
namespace StellaOps.Policy.Engine.Tests.Integration;
|
|
|
|
/// <summary>
|
|
/// Concurrent evaluation tests verifying that EWS calculation is thread-safe
|
|
/// in the policy pipeline. These tests stress-test the system under concurrent load.
|
|
/// </summary>
|
|
[Trait("Category", "Concurrency")]
|
|
[Trait("Category", "Integration")]
|
|
[Trait("Sprint", "8200.0012.0003")]
|
|
[Trait("Task", "PINT-8200-042")]
|
|
public sealed class EwsConcurrentEvaluationTests
|
|
{
|
|
private static ServiceCollection CreateServicesWithConfiguration()
|
|
{
|
|
var services = new ServiceCollection();
|
|
var configuration = new ConfigurationBuilder()
|
|
.AddInMemoryCollection()
|
|
.Build();
|
|
services.AddSingleton<IConfiguration>(configuration);
|
|
return services;
|
|
}
|
|
|
|
#region Calculator Thread Safety Tests
|
|
|
|
[Fact(DisplayName = "Calculator is thread-safe for concurrent same-input calculations")]
|
|
public async Task Calculator_IsThreadSafe_ForSameInputCalculations()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var input = CreateTestInput("concurrent-same-input");
|
|
var results = new ConcurrentBag<EvidenceWeightedScoreResult>();
|
|
|
|
// Act - Concurrent calculations with same input
|
|
var tasks = Enumerable.Range(0, 100)
|
|
.Select(_ => Task.Run(() =>
|
|
{
|
|
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
results.Add(result);
|
|
}))
|
|
.ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert - All should produce identical results
|
|
var resultList = results.ToList();
|
|
var first = resultList[0];
|
|
resultList.Should().AllSatisfy(r =>
|
|
{
|
|
r.Score.Should().Be(first.Score, "concurrent calculations must produce same score");
|
|
r.Bucket.Should().Be(first.Bucket, "concurrent calculations must produce same bucket");
|
|
});
|
|
}
|
|
|
|
[Fact(DisplayName = "Calculator is thread-safe for concurrent different-input calculations")]
|
|
public async Task Calculator_IsThreadSafe_ForDifferentInputCalculations()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var results = new ConcurrentDictionary<string, EvidenceWeightedScoreResult>();
|
|
|
|
var inputs = Enumerable.Range(0, 50)
|
|
.Select(i => CreateTestInput($"concurrent-different-{i}", i / 50.0))
|
|
.ToList();
|
|
|
|
// Act - Concurrent calculations with different inputs
|
|
var tasks = inputs.Select(input => Task.Run(() =>
|
|
{
|
|
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
results[input.FindingId] = result;
|
|
})).ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert - Each should produce valid result
|
|
results.Should().HaveCount(50);
|
|
foreach (var kvp in results)
|
|
{
|
|
kvp.Value.FindingId.Should().Be(kvp.Key);
|
|
kvp.Value.Score.Should().BeInRange(0, 100);
|
|
}
|
|
}
|
|
|
|
[Fact(DisplayName = "Calculator handles high concurrency without contention issues")]
|
|
public async Task Calculator_HandlesHighConcurrency_WithoutContention()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var errors = new ConcurrentBag<Exception>();
|
|
var results = new ConcurrentBag<EvidenceWeightedScoreResult>();
|
|
|
|
// Act - Very high concurrency (500 parallel tasks)
|
|
var tasks = Enumerable.Range(0, 500)
|
|
.Select(i => Task.Run(() =>
|
|
{
|
|
try
|
|
{
|
|
var input = CreateTestInput($"stress-test-{i}", (i % 100) / 100.0);
|
|
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
results.Add(result);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
}))
|
|
.ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert - No errors, all results valid
|
|
errors.Should().BeEmpty("no exceptions should occur under high concurrency");
|
|
results.Should().HaveCount(500);
|
|
results.Should().AllSatisfy(r => r.Score.Should().BeInRange(0, 100));
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Enricher Thread Safety Tests
|
|
|
|
[Fact(DisplayName = "Enricher is thread-safe for concurrent enrichments")]
|
|
public async Task Enricher_IsThreadSafe_ForConcurrentEnrichments()
|
|
{
|
|
// Arrange
|
|
var services = CreateServicesWithConfiguration();
|
|
services.AddEvidenceWeightedScoring();
|
|
services.AddEvidenceNormalizers();
|
|
services.AddEvidenceWeightedScore(opts =>
|
|
{
|
|
opts.Enabled = true;
|
|
opts.EnableCaching = false; // Test without caching
|
|
});
|
|
var provider = services.BuildServiceProvider();
|
|
var enricher = provider.GetRequiredService<IFindingScoreEnricher>();
|
|
|
|
var evidence = CreateTestEvidence("concurrent-enricher-test");
|
|
var results = new ConcurrentBag<ScoreEnrichmentResult>();
|
|
|
|
// Act
|
|
var tasks = Enumerable.Range(0, 100)
|
|
.Select(_ => Task.Run(() =>
|
|
{
|
|
var result = enricher.Enrich(evidence);
|
|
results.Add(result);
|
|
}))
|
|
.ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert
|
|
var resultList = results.ToList();
|
|
resultList.Should().HaveCount(100);
|
|
var first = resultList[0];
|
|
resultList.Should().AllSatisfy(r =>
|
|
{
|
|
r.Score!.Score.Should().Be(first.Score!.Score);
|
|
r.Score!.Bucket.Should().Be(first.Score!.Bucket);
|
|
});
|
|
}
|
|
|
|
[Fact(DisplayName = "Enricher with caching handles concurrent requests correctly")]
|
|
public async Task Enricher_WithCaching_HandlesConcurrentRequests()
|
|
{
|
|
// Arrange
|
|
var services = CreateServicesWithConfiguration();
|
|
services.AddEvidenceWeightedScoring();
|
|
services.AddEvidenceNormalizers();
|
|
services.AddEvidenceWeightedScore(opts =>
|
|
{
|
|
opts.Enabled = true;
|
|
opts.EnableCaching = true; // Enable caching
|
|
});
|
|
var provider = services.BuildServiceProvider();
|
|
var enricher = provider.GetRequiredService<IFindingScoreEnricher>();
|
|
|
|
var evidence = CreateTestEvidence("cached-concurrent-test");
|
|
var results = new ConcurrentBag<ScoreEnrichmentResult>();
|
|
|
|
// Act - First warm up the cache
|
|
enricher.Enrich(evidence);
|
|
|
|
// Then concurrent reads
|
|
var tasks = Enumerable.Range(0, 100)
|
|
.Select(_ => Task.Run(() =>
|
|
{
|
|
var result = enricher.Enrich(evidence);
|
|
results.Add(result);
|
|
}))
|
|
.ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert - All should be from cache and identical
|
|
var resultList = results.ToList();
|
|
resultList.Should().HaveCount(100);
|
|
resultList.Should().AllSatisfy(r => r.FromCache.Should().BeTrue("all should hit cache"));
|
|
|
|
var first = resultList[0];
|
|
resultList.Should().AllSatisfy(r =>
|
|
{
|
|
r.Score!.Score.Should().Be(first.Score!.Score);
|
|
});
|
|
}
|
|
|
|
[Fact(DisplayName = "Multiple findings enriched concurrently produce correct results")]
|
|
public async Task MultipleFindingsEnrichedConcurrently_ProduceCorrectResults()
|
|
{
|
|
// Arrange
|
|
var services = CreateServicesWithConfiguration();
|
|
services.AddEvidenceWeightedScoring();
|
|
services.AddEvidenceNormalizers();
|
|
services.AddEvidenceWeightedScore(opts =>
|
|
{
|
|
opts.Enabled = true;
|
|
opts.EnableCaching = true;
|
|
});
|
|
var provider = services.BuildServiceProvider();
|
|
var enricher = provider.GetRequiredService<IFindingScoreEnricher>();
|
|
|
|
var evidences = Enumerable.Range(0, 20)
|
|
.Select(i => CreateTestEvidence($"multi-finding-{i}", i / 20.0))
|
|
.ToList();
|
|
|
|
var results = new ConcurrentDictionary<string, ScoreEnrichmentResult>();
|
|
|
|
// Act - Enrich multiple different findings concurrently
|
|
var tasks = evidences.Select(evidence => Task.Run(() =>
|
|
{
|
|
var result = enricher.Enrich(evidence);
|
|
results[evidence.FindingId] = result;
|
|
})).ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert
|
|
results.Should().HaveCount(20);
|
|
foreach (var kvp in results)
|
|
{
|
|
kvp.Value.FindingId.Should().Be(kvp.Key);
|
|
kvp.Value.Score.Should().NotBeNull();
|
|
kvp.Value.Score!.Score.Should().BeInRange(0, 100);
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Cache Thread Safety Tests
|
|
|
|
[Fact(DisplayName = "Cache handles concurrent reads and writes safely")]
|
|
public async Task Cache_HandlesConcurrentReadsAndWrites()
|
|
{
|
|
// Arrange
|
|
var cache = new InMemoryScoreEnrichmentCache();
|
|
var readSuccesses = new ConcurrentBag<bool>();
|
|
var writes = new ConcurrentBag<bool>();
|
|
|
|
var testResult = new EvidenceWeightedScoreResult
|
|
{
|
|
FindingId = "cache-test",
|
|
Score = 75,
|
|
Bucket = ScoreBucket.ScheduleNext,
|
|
Inputs = new EvidenceInputValues(Rch: 0.8, Rts: 0.7, Bkp: 0.3, Xpl: 0.6, Src: 0.5, Mit: 0.2),
|
|
Weights = EvidenceWeights.Default,
|
|
Breakdown = [],
|
|
Flags = [],
|
|
Explanations = [],
|
|
Caps = new AppliedGuardrails(),
|
|
PolicyDigest = "test-digest",
|
|
CalculatedAt = DateTimeOffset.UtcNow
|
|
};
|
|
|
|
// Act - Mixed concurrent reads and writes
|
|
var tasks = Enumerable.Range(0, 200)
|
|
.Select(i => Task.Run(() =>
|
|
{
|
|
if (i % 3 == 0)
|
|
{
|
|
// Write
|
|
cache.Set($"finding-{i % 10}", testResult);
|
|
writes.Add(true);
|
|
}
|
|
else
|
|
{
|
|
// Read
|
|
var found = cache.TryGet($"finding-{i % 10}", out _);
|
|
readSuccesses.Add(found);
|
|
}
|
|
}))
|
|
.ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert - No exceptions means thread-safe
|
|
writes.Should().NotBeEmpty();
|
|
readSuccesses.Should().NotBeEmpty();
|
|
}
|
|
|
|
[Fact(DisplayName = "Cache maintains consistency under concurrent access")]
|
|
public async Task Cache_MaintainsConsistency_UnderConcurrentAccess()
|
|
{
|
|
// Arrange
|
|
var cache = new InMemoryScoreEnrichmentCache();
|
|
var findingId = "consistency-test";
|
|
|
|
var testResult = new EvidenceWeightedScoreResult
|
|
{
|
|
FindingId = findingId,
|
|
Score = 80,
|
|
Bucket = ScoreBucket.ScheduleNext,
|
|
Inputs = new EvidenceInputValues(Rch: 0.8, Rts: 0.7, Bkp: 0.3, Xpl: 0.6, Src: 0.5, Mit: 0.2),
|
|
Weights = EvidenceWeights.Default,
|
|
Breakdown = [],
|
|
Flags = [],
|
|
Explanations = [],
|
|
Caps = new AppliedGuardrails(),
|
|
PolicyDigest = "test-digest",
|
|
CalculatedAt = DateTimeOffset.UtcNow
|
|
};
|
|
|
|
// Set initial value
|
|
cache.Set(findingId, testResult);
|
|
|
|
var readResults = new ConcurrentBag<int>();
|
|
|
|
// Act - Many concurrent reads
|
|
var tasks = Enumerable.Range(0, 500)
|
|
.Select(_ => Task.Run(() =>
|
|
{
|
|
if (cache.TryGet(findingId, out var result) && result is not null)
|
|
readResults.Add(result.Score);
|
|
}))
|
|
.ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert - All reads should get the same value
|
|
readResults.Should().OnlyContain(score => score == 80, "all reads should return consistent value");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Race Condition Tests
|
|
|
|
[Fact(DisplayName = "No race conditions in calculator under parallel execution")]
|
|
public async Task NoRaceConditions_InCalculator_UnderParallelExecution()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var inputs = Enumerable.Range(0, 100)
|
|
.Select(i => CreateTestInput($"race-test-{i}", (i % 10) / 10.0))
|
|
.ToList();
|
|
|
|
var results = new ConcurrentDictionary<string, List<int>>();
|
|
|
|
// Initialize result lists
|
|
foreach (var input in inputs)
|
|
results[input.FindingId] = new List<int>();
|
|
|
|
// Act - Each input calculated multiple times concurrently
|
|
var tasks = inputs.SelectMany(input =>
|
|
Enumerable.Range(0, 5).Select(_ => Task.Run(() =>
|
|
{
|
|
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
|
lock (results[input.FindingId])
|
|
{
|
|
results[input.FindingId].Add(result.Score);
|
|
}
|
|
})))
|
|
.ToArray();
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
// Assert - For each input, all calculations should produce same score
|
|
foreach (var kvp in results)
|
|
{
|
|
var scores = kvp.Value;
|
|
scores.Should().HaveCount(5);
|
|
scores.Distinct().Should().HaveCount(1,
|
|
$"all calculations for {kvp.Key} should produce same score, but got: {string.Join(", ", scores)}");
|
|
}
|
|
}
|
|
|
|
[Fact(DisplayName = "Policy changes between calculations don't cause race conditions")]
|
|
public async Task PolicyChanges_DontCauseRaceConditions()
|
|
{
|
|
// Arrange
|
|
var calculator = new EvidenceWeightedScoreCalculator();
|
|
var input = CreateTestInput("policy-race-test");
|
|
|
|
var policy1 = EvidenceWeightPolicy.DefaultProduction;
|
|
var policy2 = new EvidenceWeightPolicy
|
|
{
|
|
Version = "ews.v1",
|
|
Profile = "alternate",
|
|
Weights = new EvidenceWeights
|
|
{
|
|
Rch = 0.40,
|
|
Rts = 0.20,
|
|
Bkp = 0.10,
|
|
Xpl = 0.15,
|
|
Src = 0.10,
|
|
Mit = 0.05
|
|
}
|
|
};
|
|
|
|
var results1 = new ConcurrentBag<int>();
|
|
var results2 = new ConcurrentBag<int>();
|
|
|
|
// Act - Concurrent calculations with different policies
|
|
var tasks1 = Enumerable.Range(0, 50)
|
|
.Select(_ => Task.Run(() =>
|
|
{
|
|
var result = calculator.Calculate(input, policy1);
|
|
results1.Add(result.Score);
|
|
}));
|
|
|
|
var tasks2 = Enumerable.Range(0, 50)
|
|
.Select(_ => Task.Run(() =>
|
|
{
|
|
var result = calculator.Calculate(input, policy2);
|
|
results2.Add(result.Score);
|
|
}));
|
|
|
|
await Task.WhenAll(tasks1.Concat(tasks2));
|
|
|
|
// Assert - Each policy should produce consistent results
|
|
results1.Distinct().Should().HaveCount(1, "all policy1 calculations should produce same score");
|
|
results2.Distinct().Should().HaveCount(1, "all policy2 calculations should produce same score");
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Test Helpers
|
|
|
|
private static EvidenceWeightedScoreInput CreateTestInput(string findingId, double factor = 0.5)
|
|
{
|
|
return new EvidenceWeightedScoreInput
|
|
{
|
|
FindingId = findingId,
|
|
Rch = 0.50 + factor * 0.3,
|
|
Rts = 0.40 + factor * 0.3,
|
|
Bkp = 0.30 + factor * 0.2,
|
|
Xpl = 0.45 + factor * 0.3,
|
|
Src = 0.55 + factor * 0.2,
|
|
Mit = 0.15 + factor * 0.1
|
|
};
|
|
}
|
|
|
|
private static FindingEvidence CreateTestEvidence(string findingId, double factor = 0.5)
|
|
{
|
|
return new FindingEvidence
|
|
{
|
|
FindingId = findingId,
|
|
Reachability = new ReachabilityInput
|
|
{
|
|
State = StellaOps.Signals.EvidenceWeightedScore.ReachabilityState.DynamicReachable,
|
|
Confidence = 0.70 + factor * 0.2
|
|
},
|
|
Runtime = new RuntimeInput
|
|
{
|
|
Posture = StellaOps.Signals.EvidenceWeightedScore.RuntimePosture.ActiveTracing,
|
|
ObservationCount = (int)(3 + factor * 5),
|
|
RecencyFactor = 0.65 + factor * 0.2
|
|
},
|
|
Exploit = new ExploitInput
|
|
{
|
|
EpssScore = 0.35 + factor * 0.3,
|
|
EpssPercentile = (int)(60 + factor * 30),
|
|
KevStatus = factor > 0.5 ? KevStatus.InKev : KevStatus.NotInKev,
|
|
PublicExploitAvailable = factor > 0.7
|
|
}
|
|
};
|
|
}
|
|
|
|
#endregion
|
|
}
|