save development progress
This commit is contained in:
StellaOps Bot
@@ -0,0 +1,489 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
// Copyright © 2025 StellaOps
|
||||
// Sprint: SPRINT_8200_0012_0003_policy_engine_integration
|
||||
// Task: PINT-8200-042 - Concurrent evaluation test: thread-safe EWS in policy pipeline
|
||||
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Policy.Engine.Scoring.EvidenceWeightedScore;
|
||||
using StellaOps.Signals.EvidenceWeightedScore;
|
||||
using StellaOps.Signals.EvidenceWeightedScore.Normalizers;
|
||||
using System.Collections.Concurrent;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Policy.Engine.Tests.Integration;
|
||||
|
||||
/// <summary>
|
||||
/// Concurrent evaluation tests verifying that EWS calculation is thread-safe
|
||||
/// in the policy pipeline. These tests stress-test the system under concurrent load.
|
||||
/// </summary>
|
||||
[Trait("Category", "Concurrency")]
|
||||
[Trait("Category", "Integration")]
|
||||
[Trait("Sprint", "8200.0012.0003")]
|
||||
[Trait("Task", "PINT-8200-042")]
|
||||
public sealed class EwsConcurrentEvaluationTests
|
||||
{
|
||||
private static ServiceCollection CreateServicesWithConfiguration()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection()
|
||||
.Build();
|
||||
services.AddSingleton<IConfiguration>(configuration);
|
||||
return services;
|
||||
}
|
||||
|
||||
#region Calculator Thread Safety Tests
|
||||
|
||||
[Fact(DisplayName = "Calculator is thread-safe for concurrent same-input calculations")]
|
||||
public async Task Calculator_IsThreadSafe_ForSameInputCalculations()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new EvidenceWeightedScoreCalculator();
|
||||
var input = CreateTestInput("concurrent-same-input");
|
||||
var results = new ConcurrentBag<EvidenceWeightedScoreResult>();
|
||||
|
||||
// Act - Concurrent calculations with same input
|
||||
var tasks = Enumerable.Range(0, 100)
|
||||
.Select(_ => Task.Run(() =>
|
||||
{
|
||||
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
||||
results.Add(result);
|
||||
}))
|
||||
.ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - All should produce identical results
|
||||
var resultList = results.ToList();
|
||||
var first = resultList[0];
|
||||
resultList.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Score.Should().Be(first.Score, "concurrent calculations must produce same score");
|
||||
r.Bucket.Should().Be(first.Bucket, "concurrent calculations must produce same bucket");
|
||||
});
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Calculator is thread-safe for concurrent different-input calculations")]
|
||||
public async Task Calculator_IsThreadSafe_ForDifferentInputCalculations()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new EvidenceWeightedScoreCalculator();
|
||||
var results = new ConcurrentDictionary<string, EvidenceWeightedScoreResult>();
|
||||
|
||||
var inputs = Enumerable.Range(0, 50)
|
||||
.Select(i => CreateTestInput($"concurrent-different-{i}", i / 50.0))
|
||||
.ToList();
|
||||
|
||||
// Act - Concurrent calculations with different inputs
|
||||
var tasks = inputs.Select(input => Task.Run(() =>
|
||||
{
|
||||
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
||||
results[input.FindingId] = result;
|
||||
})).ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - Each should produce valid result
|
||||
results.Should().HaveCount(50);
|
||||
foreach (var kvp in results)
|
||||
{
|
||||
kvp.Value.FindingId.Should().Be(kvp.Key);
|
||||
kvp.Value.Score.Should().BeInRange(0, 100);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Calculator handles high concurrency without contention issues")]
|
||||
public async Task Calculator_HandlesHighConcurrency_WithoutContention()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new EvidenceWeightedScoreCalculator();
|
||||
var errors = new ConcurrentBag<Exception>();
|
||||
var results = new ConcurrentBag<EvidenceWeightedScoreResult>();
|
||||
|
||||
// Act - Very high concurrency (500 parallel tasks)
|
||||
var tasks = Enumerable.Range(0, 500)
|
||||
.Select(i => Task.Run(() =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var input = CreateTestInput($"stress-test-{i}", (i % 100) / 100.0);
|
||||
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
||||
results.Add(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
errors.Add(ex);
|
||||
}
|
||||
}))
|
||||
.ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - No errors, all results valid
|
||||
errors.Should().BeEmpty("no exceptions should occur under high concurrency");
|
||||
results.Should().HaveCount(500);
|
||||
results.Should().AllSatisfy(r => r.Score.Should().BeInRange(0, 100));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Enricher Thread Safety Tests
|
||||
|
||||
[Fact(DisplayName = "Enricher is thread-safe for concurrent enrichments")]
|
||||
public async Task Enricher_IsThreadSafe_ForConcurrentEnrichments()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateServicesWithConfiguration();
|
||||
services.AddEvidenceWeightedScoring();
|
||||
services.AddEvidenceNormalizers();
|
||||
services.AddEvidenceWeightedScore(opts =>
|
||||
{
|
||||
opts.Enabled = true;
|
||||
opts.EnableCaching = false; // Test without caching
|
||||
});
|
||||
var provider = services.BuildServiceProvider();
|
||||
var enricher = provider.GetRequiredService<IFindingScoreEnricher>();
|
||||
|
||||
var evidence = CreateTestEvidence("concurrent-enricher-test");
|
||||
var results = new ConcurrentBag<ScoreEnrichmentResult>();
|
||||
|
||||
// Act
|
||||
var tasks = Enumerable.Range(0, 100)
|
||||
.Select(_ => Task.Run(() =>
|
||||
{
|
||||
var result = enricher.Enrich(evidence);
|
||||
results.Add(result);
|
||||
}))
|
||||
.ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert
|
||||
var resultList = results.ToList();
|
||||
resultList.Should().HaveCount(100);
|
||||
var first = resultList[0];
|
||||
resultList.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Score!.Score.Should().Be(first.Score!.Score);
|
||||
r.Score!.Bucket.Should().Be(first.Score!.Bucket);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Enricher with caching handles concurrent requests correctly")]
|
||||
public async Task Enricher_WithCaching_HandlesConcurrentRequests()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateServicesWithConfiguration();
|
||||
services.AddEvidenceWeightedScoring();
|
||||
services.AddEvidenceNormalizers();
|
||||
services.AddEvidenceWeightedScore(opts =>
|
||||
{
|
||||
opts.Enabled = true;
|
||||
opts.EnableCaching = true; // Enable caching
|
||||
});
|
||||
var provider = services.BuildServiceProvider();
|
||||
var enricher = provider.GetRequiredService<IFindingScoreEnricher>();
|
||||
|
||||
var evidence = CreateTestEvidence("cached-concurrent-test");
|
||||
var results = new ConcurrentBag<ScoreEnrichmentResult>();
|
||||
|
||||
// Act - First warm up the cache
|
||||
enricher.Enrich(evidence);
|
||||
|
||||
// Then concurrent reads
|
||||
var tasks = Enumerable.Range(0, 100)
|
||||
.Select(_ => Task.Run(() =>
|
||||
{
|
||||
var result = enricher.Enrich(evidence);
|
||||
results.Add(result);
|
||||
}))
|
||||
.ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - All should be from cache and identical
|
||||
var resultList = results.ToList();
|
||||
resultList.Should().HaveCount(100);
|
||||
resultList.Should().AllSatisfy(r => r.FromCache.Should().BeTrue("all should hit cache"));
|
||||
|
||||
var first = resultList[0];
|
||||
resultList.Should().AllSatisfy(r =>
|
||||
{
|
||||
r.Score!.Score.Should().Be(first.Score!.Score);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Multiple findings enriched concurrently produce correct results")]
|
||||
public async Task MultipleFindingsEnrichedConcurrently_ProduceCorrectResults()
|
||||
{
|
||||
// Arrange
|
||||
var services = CreateServicesWithConfiguration();
|
||||
services.AddEvidenceWeightedScoring();
|
||||
services.AddEvidenceNormalizers();
|
||||
services.AddEvidenceWeightedScore(opts =>
|
||||
{
|
||||
opts.Enabled = true;
|
||||
opts.EnableCaching = true;
|
||||
});
|
||||
var provider = services.BuildServiceProvider();
|
||||
var enricher = provider.GetRequiredService<IFindingScoreEnricher>();
|
||||
|
||||
var evidences = Enumerable.Range(0, 20)
|
||||
.Select(i => CreateTestEvidence($"multi-finding-{i}", i / 20.0))
|
||||
.ToList();
|
||||
|
||||
var results = new ConcurrentDictionary<string, ScoreEnrichmentResult>();
|
||||
|
||||
// Act - Enrich multiple different findings concurrently
|
||||
var tasks = evidences.Select(evidence => Task.Run(() =>
|
||||
{
|
||||
var result = enricher.Enrich(evidence);
|
||||
results[evidence.FindingId] = result;
|
||||
})).ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert
|
||||
results.Should().HaveCount(20);
|
||||
foreach (var kvp in results)
|
||||
{
|
||||
kvp.Value.FindingId.Should().Be(kvp.Key);
|
||||
kvp.Value.Score.Should().NotBeNull();
|
||||
kvp.Value.Score!.Score.Should().BeInRange(0, 100);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Cache Thread Safety Tests
|
||||
|
||||
[Fact(DisplayName = "Cache handles concurrent reads and writes safely")]
|
||||
public async Task Cache_HandlesConcurrentReadsAndWrites()
|
||||
{
|
||||
// Arrange
|
||||
var cache = new InMemoryScoreEnrichmentCache();
|
||||
var readSuccesses = new ConcurrentBag<bool>();
|
||||
var writes = new ConcurrentBag<bool>();
|
||||
|
||||
var testResult = new EvidenceWeightedScoreResult
|
||||
{
|
||||
FindingId = "cache-test",
|
||||
Score = 75,
|
||||
Bucket = ScoreBucket.ScheduleNext,
|
||||
Inputs = new EvidenceInputValues(Rch: 0.8, Rts: 0.7, Bkp: 0.3, Xpl: 0.6, Src: 0.5, Mit: 0.2),
|
||||
Weights = EvidenceWeights.Default,
|
||||
Breakdown = [],
|
||||
Flags = [],
|
||||
Explanations = [],
|
||||
Caps = new AppliedGuardrails(),
|
||||
PolicyDigest = "test-digest",
|
||||
CalculatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
// Act - Mixed concurrent reads and writes
|
||||
var tasks = Enumerable.Range(0, 200)
|
||||
.Select(i => Task.Run(() =>
|
||||
{
|
||||
if (i % 3 == 0)
|
||||
{
|
||||
// Write
|
||||
cache.Set($"finding-{i % 10}", testResult);
|
||||
writes.Add(true);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Read
|
||||
var found = cache.TryGet($"finding-{i % 10}", out _);
|
||||
readSuccesses.Add(found);
|
||||
}
|
||||
}))
|
||||
.ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - No exceptions means thread-safe
|
||||
writes.Should().NotBeEmpty();
|
||||
readSuccesses.Should().NotBeEmpty();
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Cache maintains consistency under concurrent access")]
|
||||
public async Task Cache_MaintainsConsistency_UnderConcurrentAccess()
|
||||
{
|
||||
// Arrange
|
||||
var cache = new InMemoryScoreEnrichmentCache();
|
||||
var findingId = "consistency-test";
|
||||
|
||||
var testResult = new EvidenceWeightedScoreResult
|
||||
{
|
||||
FindingId = findingId,
|
||||
Score = 80,
|
||||
Bucket = ScoreBucket.ScheduleNext,
|
||||
Inputs = new EvidenceInputValues(Rch: 0.8, Rts: 0.7, Bkp: 0.3, Xpl: 0.6, Src: 0.5, Mit: 0.2),
|
||||
Weights = EvidenceWeights.Default,
|
||||
Breakdown = [],
|
||||
Flags = [],
|
||||
Explanations = [],
|
||||
Caps = new AppliedGuardrails(),
|
||||
PolicyDigest = "test-digest",
|
||||
CalculatedAt = DateTimeOffset.UtcNow
|
||||
};
|
||||
|
||||
// Set initial value
|
||||
cache.Set(findingId, testResult);
|
||||
|
||||
var readResults = new ConcurrentBag<int>();
|
||||
|
||||
// Act - Many concurrent reads
|
||||
var tasks = Enumerable.Range(0, 500)
|
||||
.Select(_ => Task.Run(() =>
|
||||
{
|
||||
if (cache.TryGet(findingId, out var result) && result is not null)
|
||||
readResults.Add(result.Score);
|
||||
}))
|
||||
.ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - All reads should get the same value
|
||||
readResults.Should().OnlyContain(score => score == 80, "all reads should return consistent value");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Race Condition Tests
|
||||
|
||||
[Fact(DisplayName = "No race conditions in calculator under parallel execution")]
|
||||
public async Task NoRaceConditions_InCalculator_UnderParallelExecution()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new EvidenceWeightedScoreCalculator();
|
||||
var inputs = Enumerable.Range(0, 100)
|
||||
.Select(i => CreateTestInput($"race-test-{i}", (i % 10) / 10.0))
|
||||
.ToList();
|
||||
|
||||
var results = new ConcurrentDictionary<string, List<int>>();
|
||||
|
||||
// Initialize result lists
|
||||
foreach (var input in inputs)
|
||||
results[input.FindingId] = new List<int>();
|
||||
|
||||
// Act - Each input calculated multiple times concurrently
|
||||
var tasks = inputs.SelectMany(input =>
|
||||
Enumerable.Range(0, 5).Select(_ => Task.Run(() =>
|
||||
{
|
||||
var result = calculator.Calculate(input, EvidenceWeightPolicy.DefaultProduction);
|
||||
lock (results[input.FindingId])
|
||||
{
|
||||
results[input.FindingId].Add(result.Score);
|
||||
}
|
||||
})))
|
||||
.ToArray();
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// Assert - For each input, all calculations should produce same score
|
||||
foreach (var kvp in results)
|
||||
{
|
||||
var scores = kvp.Value;
|
||||
scores.Should().HaveCount(5);
|
||||
scores.Distinct().Should().HaveCount(1,
|
||||
$"all calculations for {kvp.Key} should produce same score, but got: {string.Join(", ", scores)}");
|
||||
}
|
||||
}
|
||||
|
||||
[Fact(DisplayName = "Policy changes between calculations don't cause race conditions")]
|
||||
public async Task PolicyChanges_DontCauseRaceConditions()
|
||||
{
|
||||
// Arrange
|
||||
var calculator = new EvidenceWeightedScoreCalculator();
|
||||
var input = CreateTestInput("policy-race-test");
|
||||
|
||||
var policy1 = EvidenceWeightPolicy.DefaultProduction;
|
||||
var policy2 = new EvidenceWeightPolicy
|
||||
{
|
||||
Version = "ews.v1",
|
||||
Profile = "alternate",
|
||||
Weights = new EvidenceWeights
|
||||
{
|
||||
Rch = 0.40,
|
||||
Rts = 0.20,
|
||||
Bkp = 0.10,
|
||||
Xpl = 0.15,
|
||||
Src = 0.10,
|
||||
Mit = 0.05
|
||||
}
|
||||
};
|
||||
|
||||
var results1 = new ConcurrentBag<int>();
|
||||
var results2 = new ConcurrentBag<int>();
|
||||
|
||||
// Act - Concurrent calculations with different policies
|
||||
var tasks1 = Enumerable.Range(0, 50)
|
||||
.Select(_ => Task.Run(() =>
|
||||
{
|
||||
var result = calculator.Calculate(input, policy1);
|
||||
results1.Add(result.Score);
|
||||
}));
|
||||
|
||||
var tasks2 = Enumerable.Range(0, 50)
|
||||
.Select(_ => Task.Run(() =>
|
||||
{
|
||||
var result = calculator.Calculate(input, policy2);
|
||||
results2.Add(result.Score);
|
||||
}));
|
||||
|
||||
await Task.WhenAll(tasks1.Concat(tasks2));
|
||||
|
||||
// Assert - Each policy should produce consistent results
|
||||
results1.Distinct().Should().HaveCount(1, "all policy1 calculations should produce same score");
|
||||
results2.Distinct().Should().HaveCount(1, "all policy2 calculations should produce same score");
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Test Helpers
|
||||
|
||||
private static EvidenceWeightedScoreInput CreateTestInput(string findingId, double factor = 0.5)
|
||||
{
|
||||
return new EvidenceWeightedScoreInput
|
||||
{
|
||||
FindingId = findingId,
|
||||
Rch = 0.50 + factor * 0.3,
|
||||
Rts = 0.40 + factor * 0.3,
|
||||
Bkp = 0.30 + factor * 0.2,
|
||||
Xpl = 0.45 + factor * 0.3,
|
||||
Src = 0.55 + factor * 0.2,
|
||||
Mit = 0.15 + factor * 0.1
|
||||
};
|
||||
}
|
||||
|
||||
private static FindingEvidence CreateTestEvidence(string findingId, double factor = 0.5)
|
||||
{
|
||||
return new FindingEvidence
|
||||
{
|
||||
FindingId = findingId,
|
||||
Reachability = new ReachabilityInput
|
||||
{
|
||||
State = StellaOps.Signals.EvidenceWeightedScore.ReachabilityState.DynamicReachable,
|
||||
Confidence = 0.70 + factor * 0.2
|
||||
},
|
||||
Runtime = new RuntimeInput
|
||||
{
|
||||
Posture = StellaOps.Signals.EvidenceWeightedScore.RuntimePosture.ActiveTracing,
|
||||
ObservationCount = (int)(3 + factor * 5),
|
||||
RecencyFactor = 0.65 + factor * 0.2
|
||||
},
|
||||
Exploit = new ExploitInput
|
||||
{
|
||||
EpssScore = 0.35 + factor * 0.3,
|
||||
EpssPercentile = (int)(60 + factor * 30),
|
||||
KevStatus = factor > 0.5 ? KevStatus.InKev : KevStatus.NotInKev,
|
||||
PublicExploitAvailable = factor > 0.7
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
Reference in New Issue
Block a user