Refactor code structure and optimize performance across multiple modules
This commit is contained in:
@@ -15,6 +15,8 @@ using StellaOps.AdvisoryAI.Orchestration;
|
||||
using StellaOps.AdvisoryAI.Prompting;
|
||||
using Xunit;
|
||||
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryGuardrailInjectionTests
|
||||
@@ -35,7 +37,8 @@ public sealed class AdvisoryGuardrailInjectionTests
|
||||
public static IEnumerable<object[]> InjectionPayloads =>
|
||||
HarnessCases.Value.Select(testCase => new object[] { testCase });
|
||||
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[MemberData(nameof(InjectionPayloads))]
|
||||
public async Task EvaluateAsync_CompliesWithGuardrailHarness(InjectionCase testCase)
|
||||
{
|
||||
|
||||
@@ -12,11 +12,14 @@ using StellaOps.AdvisoryAI.Guardrails;
|
||||
using StellaOps.AdvisoryAI.Hosting;
|
||||
using Xunit;
|
||||
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryGuardrailOptionsBindingTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task AddAdvisoryAiCore_ConfiguresGuardrailOptionsFromServiceOptions()
|
||||
{
|
||||
var tempRoot = CreateTempDirectory();
|
||||
@@ -47,7 +50,8 @@ public sealed class AdvisoryGuardrailOptionsBindingTests
|
||||
options.BlockedPhrases.Should().Contain("dump cache");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task AddAdvisoryAiCore_ThrowsWhenPhraseFileMissing()
|
||||
{
|
||||
var tempRoot = CreateTempDirectory();
|
||||
|
||||
@@ -16,6 +16,8 @@ using StellaOps.AdvisoryAI.Orchestration;
|
||||
using StellaOps.AdvisoryAI.Prompting;
|
||||
using Xunit;
|
||||
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryGuardrailPerformanceTests
|
||||
@@ -27,7 +29,8 @@ public sealed class AdvisoryGuardrailPerformanceTests
|
||||
|
||||
public static IEnumerable<object[]> PerfScenarios => LoadPerfScenarios();
|
||||
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[MemberData(nameof(PerfScenarios))]
|
||||
public async Task EvaluateAsync_CompletesWithinBudget(PerfScenario scenario)
|
||||
{
|
||||
@@ -53,7 +56,8 @@ public sealed class AdvisoryGuardrailPerformanceTests
|
||||
$"{scenario.Name} exceeded the allotted {scenario.MaxDurationMs} ms budget (measured {stopwatch.ElapsedMilliseconds} ms)");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_HonorsSeededBlockedPhrases()
|
||||
{
|
||||
var phrases = LoadSeededBlockedPhrases();
|
||||
|
||||
@@ -7,11 +7,13 @@ using StellaOps.AdvisoryAI.Orchestration;
|
||||
using StellaOps.AdvisoryAI.Prompting;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryGuardrailPipelineTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_BlocksWhenCitationsMissing()
|
||||
{
|
||||
var options = Options.Create(new AdvisoryGuardrailOptions { RequireCitations = true });
|
||||
@@ -31,7 +33,8 @@ public sealed class AdvisoryGuardrailPipelineTests
|
||||
Assert.Contains(result.Violations, violation => violation.Code == "citation_missing");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task EvaluateAsync_RedactsSecrets()
|
||||
{
|
||||
var options = Options.Create(new AdvisoryGuardrailOptions());
|
||||
|
||||
@@ -17,13 +17,16 @@ using StellaOps.AdvisoryAI.Tools;
|
||||
using StellaOps.AdvisoryAI.Inference;
|
||||
using Xunit;
|
||||
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryPipelineExecutorTests : IDisposable
|
||||
{
|
||||
private readonly StubMeterFactory _meterFactory = new();
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_SavesOutputAndProvenance()
|
||||
{
|
||||
var plan = BuildMinimalPlan(cacheKey: "CACHE-1");
|
||||
@@ -49,7 +52,8 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
|
||||
saved.Guardrail.Metadata.Should().ContainKey("prompt_length");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_PersistsGuardrailOutcome()
|
||||
{
|
||||
var plan = BuildMinimalPlan(cacheKey: "CACHE-2");
|
||||
@@ -71,7 +75,8 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
|
||||
saved.Prompt.Should().Be("{\"prompt\":\"value\"}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_RecordsTelemetryMeasurements()
|
||||
{
|
||||
using var listener = new MeterListener();
|
||||
@@ -124,7 +129,8 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
|
||||
Math.Abs(measurement.Value - 1d) < 0.0001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_ComputesPartialCitationCoverage()
|
||||
{
|
||||
using var listener = new MeterListener();
|
||||
@@ -163,7 +169,8 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
|
||||
Math.Abs(measurement.Value - 0.5d) < 0.0001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_RecordsInferenceMetadata()
|
||||
{
|
||||
var plan = BuildMinimalPlan(cacheKey: "CACHE-4");
|
||||
|
||||
@@ -13,11 +13,13 @@ using StellaOps.AdvisoryAI.Orchestration;
|
||||
using StellaOps.AdvisoryAI.Tools;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryPipelineOrchestratorTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePlanAsync_ComposesDeterministicPlan()
|
||||
{
|
||||
var structuredRetriever = new FakeStructuredRetriever();
|
||||
@@ -63,7 +65,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
|
||||
Assert.Equal(plan.CacheKey, secondPlan.CacheKey);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePlanAsync_RemainsDeterministicAcrossMultipleRuns()
|
||||
{
|
||||
var structuredRetriever = new ShufflingStructuredRetriever();
|
||||
@@ -116,7 +119,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePlanAsync_PopulatesMetadataCountsFromEvidence()
|
||||
{
|
||||
var structuredRetriever = new FakeStructuredRetriever();
|
||||
@@ -156,7 +160,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
|
||||
metadata["sbom_blast_impacted_workloads"].Should().Be("3");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePlanAsync_WhenArtifactIdMissing_SkipsSbomContext()
|
||||
{
|
||||
var structuredRetriever = new FakeStructuredRetriever();
|
||||
@@ -188,7 +193,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
|
||||
Assert.DoesNotContain("sbom_dependency_path_count", plan.Metadata.Keys);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePlanAsync_RespectsOptionFlagsAndProducesStableCacheKey()
|
||||
{
|
||||
var structuredRetriever = new FakeStructuredRetriever();
|
||||
@@ -227,7 +233,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
|
||||
Assert.DoesNotContain(planOne.Metadata.Keys, key => key.StartsWith("sbom_blast_", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePlanAsync_RemainsDeterministicWhenRetrieverOrderChanges()
|
||||
{
|
||||
var structuredRetriever = new ShufflingStructuredRetriever();
|
||||
|
||||
@@ -10,11 +10,13 @@ using StellaOps.AdvisoryAI.Orchestration;
|
||||
using StellaOps.AdvisoryAI.Tools;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryPipelinePlanResponseTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void FromPlan_ProjectsMetadataAndCounts()
|
||||
{
|
||||
var request = new AdvisoryTaskRequest(AdvisoryTaskType.Summary, "adv-key");
|
||||
|
||||
@@ -15,11 +15,13 @@ using StellaOps.AdvisoryAI.Tools;
|
||||
using StellaOps.AdvisoryAI.Tests.TestUtilities;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryPlanCacheTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SetAndRetrieve_ReturnsCachedPlan()
|
||||
{
|
||||
var timeProvider = new DeterministicTimeProvider(DateTimeOffset.UtcNow);
|
||||
@@ -34,7 +36,8 @@ public sealed class AdvisoryPlanCacheTests
|
||||
retrieved.Metadata.Should().ContainKey("task_type");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ExpiredEntries_AreEvicted()
|
||||
{
|
||||
var start = DateTimeOffset.UtcNow;
|
||||
@@ -49,7 +52,8 @@ public sealed class AdvisoryPlanCacheTests
|
||||
retrieved.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SetAsync_ReplacesPlanAndRefreshesExpiration()
|
||||
{
|
||||
var timeProvider = new DeterministicTimeProvider(DateTimeOffset.UtcNow);
|
||||
@@ -69,7 +73,8 @@ public sealed class AdvisoryPlanCacheTests
|
||||
retrieved!.Request.AdvisoryKey.Should().Be("ADV-999");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SetAsync_WithInterleavedKeysRemainsDeterministic()
|
||||
{
|
||||
var timeProvider = new DeterministicTimeProvider(DateTimeOffset.UtcNow);
|
||||
@@ -108,7 +113,8 @@ public sealed class AdvisoryPlanCacheTests
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData(7)]
|
||||
[InlineData(42)]
|
||||
[InlineData(512)]
|
||||
|
||||
@@ -14,6 +14,8 @@ using StellaOps.AdvisoryAI.Tools;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryPromptAssemblerTests
|
||||
@@ -25,7 +27,8 @@ public sealed class AdvisoryPromptAssemblerTests
|
||||
_output = output;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task AssembleAsync_ProducesDeterministicPrompt()
|
||||
{
|
||||
var plan = BuildPlan();
|
||||
@@ -43,7 +46,8 @@ public sealed class AdvisoryPromptAssemblerTests
|
||||
await AssertPromptMatchesGoldenAsync("summary-prompt.json", prompt.Prompt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task AssembleAsync_ProducesConflictPromptGolden()
|
||||
{
|
||||
var plan = BuildPlan(AdvisoryTaskType.Conflict);
|
||||
@@ -56,7 +60,8 @@ public sealed class AdvisoryPromptAssemblerTests
|
||||
prompt.Metadata["task_type"].Should().Be(nameof(AdvisoryTaskType.Conflict));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task AssembleAsync_TruncatesVectorPreviewsToMaintainPromptSize()
|
||||
{
|
||||
var longPreview = new string('A', 700);
|
||||
|
||||
@@ -7,11 +7,13 @@ using StellaOps.AdvisoryAI.Documents;
|
||||
using StellaOps.AdvisoryAI.Retrievers;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryStructuredRetrieverTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RetrieveAsync_ReturnsCsafChunksWithMetadata()
|
||||
{
|
||||
var provider = CreateProvider(
|
||||
@@ -33,7 +35,8 @@ public sealed class AdvisoryStructuredRetrieverTests
|
||||
result.Chunks.Any(c => c.Section == "document.notes").Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RetrieveAsync_ReturnsOsvChunksWithAffectedMetadata()
|
||||
{
|
||||
var provider = CreateProvider(
|
||||
@@ -53,7 +56,8 @@ public sealed class AdvisoryStructuredRetrieverTests
|
||||
result.Chunks.First(c => c.Section.StartsWith("affected", StringComparison.OrdinalIgnoreCase)).Metadata.Should().ContainKey("package");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RetrieveAsync_ReturnsOpenVexChunksWithStatusMetadata()
|
||||
{
|
||||
var provider = CreateProvider(
|
||||
@@ -73,7 +77,8 @@ public sealed class AdvisoryStructuredRetrieverTests
|
||||
result.Chunks.Should().AllSatisfy(chunk => chunk.Section.Should().Be("vex.statements"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RetrieveAsync_FiltersToPreferredSections()
|
||||
{
|
||||
var provider = CreateProvider(
|
||||
|
||||
@@ -7,11 +7,13 @@ using StellaOps.AdvisoryAI.Orchestration;
|
||||
using StellaOps.AdvisoryAI.Queue;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryTaskQueueTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task EnqueueAndDequeue_ReturnsMessageInOrder()
|
||||
{
|
||||
var options = Options.Create(new AdvisoryTaskQueueOptions { Capacity = 10, DequeueWaitInterval = TimeSpan.FromMilliseconds(50) });
|
||||
|
||||
@@ -7,11 +7,13 @@ using StellaOps.AdvisoryAI.Tests.TestUtilities;
|
||||
using StellaOps.AdvisoryAI.Vectorization;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class AdvisoryVectorRetrieverTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SearchAsync_ReturnsBestMatchingChunk()
|
||||
{
|
||||
var advisoryContent = """
|
||||
|
||||
@@ -8,11 +8,13 @@ using StellaOps.Concelier.Core.Raw;
|
||||
using StellaOps.Concelier.RawModels;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class ConcelierAdvisoryDocumentProviderTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GetDocumentsAsync_ReturnsMappedDocuments()
|
||||
{
|
||||
var rawDocument = RawDocumentFactory.CreateAdvisory(
|
||||
|
||||
@@ -5,11 +5,13 @@ using StellaOps.AdvisoryAI.Context;
|
||||
using StellaOps.AdvisoryAI.Tools;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class DeterministicToolsetTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void AnalyzeDependencies_ComputesRuntimeAndDevelopmentCounts()
|
||||
{
|
||||
var context = SbomContextResult.Create(
|
||||
@@ -52,7 +54,8 @@ public sealed class DeterministicToolsetTests
|
||||
libB.DevelopmentOccurrences.Should().Be(1);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("semver", "1.2.3", "1.2.4", -1)]
|
||||
[InlineData("semver", "1.2.3", "1.2.3", 0)]
|
||||
[InlineData("semver", "1.2.4", "1.2.3", 1)]
|
||||
@@ -66,7 +69,8 @@ public sealed class DeterministicToolsetTests
|
||||
comparison.Should().Be(expected);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("semver", "1.2.3", ">=1.0.0 <2.0.0")]
|
||||
[InlineData("semver", "2.0.0", ">=2.0.0")]
|
||||
[InlineData("evr", "0:1.2-3", ">=0:1.0-0 <0:2.0-0")]
|
||||
|
||||
@@ -9,11 +9,13 @@ using StellaOps.Excititor.Core;
|
||||
using StellaOps.Excititor.Core.Observations;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class ExcititorVexDocumentProviderTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GetDocumentsAsync_ReturnsMappedObservation()
|
||||
{
|
||||
const string vulnerabilityId = "CVE-2024-9999";
|
||||
|
||||
@@ -0,0 +1,542 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using FluentAssertions;
|
||||
using StellaOps.AdvisoryAI.Explanation;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for explanation generation with mocked LLM and evidence anchoring validation.
|
||||
/// Sprint: SPRINT_20251226_015_AI_zastava_companion
|
||||
/// Task: ZASTAVA-19
|
||||
/// </summary>
|
||||
public sealed class ExplanationGeneratorIntegrationTests
|
||||
{
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_WithFullEvidence_ProducesEvidenceBackedExplanation()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "This is a test explanation with [citation:ev-001] and [citation:ev-002].",
|
||||
confidence: 0.95);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Full);
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
result.Should().NotBeNull();
|
||||
result.ExplanationId.Should().StartWith("sha256:");
|
||||
result.Authority.Should().Be(ExplanationAuthority.EvidenceBacked);
|
||||
result.CitationRate.Should().BeGreaterOrEqualTo(0.8);
|
||||
result.Citations.Should().NotBeEmpty();
|
||||
result.EvidenceRefs.Should().NotBeEmpty();
|
||||
result.InputHashes.Should().HaveCount(3);
|
||||
result.OutputHash.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_WithLowCitationRate_ProducesSuggestionExplanation()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateMinimalEvidenceContext());
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "This is a speculative explanation without proper citations.",
|
||||
confidence: 0.6);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.3);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Why);
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
result.Authority.Should().Be(ExplanationAuthority.Suggestion);
|
||||
result.CitationRate.Should().BeLessThan(0.8);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_StoresResultForReplay()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "Stored explanation [citation:ev-001].",
|
||||
confidence: 0.9);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.85);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.What);
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
var stored = await store.GetAsync(result.ExplanationId, CancellationToken.None);
|
||||
stored.Should().NotBeNull();
|
||||
stored!.ExplanationId.Should().Be(result.ExplanationId);
|
||||
stored.OutputHash.Should().Be(result.OutputHash);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_ComputesConsistentInputHashes()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateFullEvidenceContext();
|
||||
var evidenceService = new StubEvidenceRetrievalService(evidenceContext);
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "Consistent explanation.",
|
||||
confidence: 0.88);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.85);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Evidence);
|
||||
|
||||
// Act
|
||||
var result1 = await generator.GenerateAsync(request);
|
||||
var result2 = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert - same inputs should produce same input hashes
|
||||
result1.InputHashes.Should().BeEquivalentTo(result2.InputHashes);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_ProducesValidExplanationId()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "Test explanation content.",
|
||||
confidence: 0.9);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Full);
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
result.ExplanationId.Should().StartWith("sha256:");
|
||||
result.ExplanationId.Length.Should().Be(7 + 64); // "sha256:" + 64 hex chars
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_IncludesAllEvidenceRefs()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateFullEvidenceContext();
|
||||
var evidenceService = new StubEvidenceRetrievalService(evidenceContext);
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "Explanation with evidence.",
|
||||
confidence: 0.9);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Full);
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
var allEvidenceIds = evidenceContext.AllEvidence.Select(e => e.Id).ToList();
|
||||
result.EvidenceRefs.Should().BeEquivalentTo(allEvidenceIds);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_RecordsModelIdAndTemplateVersion()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
|
||||
var promptService = new StubExplanationPromptService(templateVersion: "explain-v2.1");
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "Test.",
|
||||
confidence: 0.9,
|
||||
modelId: "claude:claude-3-opus:20240229");
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Full);
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
result.ModelId.Should().Be("claude:claude-3-opus:20240229");
|
||||
result.PromptTemplateVersion.Should().Be("explain-v2.1");
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_GeneratesValidSummary()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "Detailed explanation content.",
|
||||
confidence: 0.9);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Full);
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
result.Summary.Should().NotBeNull();
|
||||
result.Summary.Line1.Should().NotBeNullOrEmpty();
|
||||
result.Summary.Line2.Should().NotBeNullOrEmpty();
|
||||
result.Summary.Line3.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData(ExplanationType.What)]
|
||||
[InlineData(ExplanationType.Why)]
|
||||
[InlineData(ExplanationType.Evidence)]
|
||||
[InlineData(ExplanationType.Counterfactual)]
|
||||
[InlineData(ExplanationType.Full)]
|
||||
public async Task GenerateAsync_HandlesAllExplanationTypes(ExplanationType type)
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: $"Explanation for {type}.",
|
||||
confidence: 0.9);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.85);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(type);
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
result.Should().NotBeNull();
|
||||
result.Content.Should().Contain(type.ToString());
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ValidateAsync_ReturnsTrueForValidEvidence()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext(), validateResult: true);
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "Test.",
|
||||
confidence: 0.9);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Full);
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Act
|
||||
var isValid = await generator.ValidateAsync(result);
|
||||
|
||||
// Assert
|
||||
isValid.Should().BeTrue();
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ValidateAsync_ReturnsFalseWhenEvidenceChanged()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext(), validateResult: false);
|
||||
var promptService = new StubExplanationPromptService();
|
||||
var inferenceClient = new StubExplanationInferenceClient(
|
||||
content: "Test.",
|
||||
confidence: 0.9);
|
||||
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
|
||||
var store = new InMemoryExplanationStore();
|
||||
|
||||
var generator = new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
|
||||
var request = CreateExplanationRequest(ExplanationType.Full);
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Act
|
||||
var isValid = await generator.ValidateAsync(result);
|
||||
|
||||
// Assert
|
||||
isValid.Should().BeFalse();
|
||||
}
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static ExplanationRequest CreateExplanationRequest(ExplanationType type) => new()
|
||||
{
|
||||
FindingId = "finding-001",
|
||||
ArtifactDigest = "sha256:abc123",
|
||||
Scope = "image",
|
||||
ScopeId = "my-image:latest",
|
||||
ExplanationType = type,
|
||||
VulnerabilityId = "CVE-2024-1234",
|
||||
ComponentPurl = "pkg:npm/lodash@4.17.20",
|
||||
PlainLanguage = false,
|
||||
MaxLength = 0,
|
||||
CorrelationId = "corr-001"
|
||||
};
|
||||
|
||||
private static EvidenceContext CreateFullEvidenceContext() => new()
|
||||
{
|
||||
SbomEvidence =
|
||||
[
|
||||
new EvidenceNode
|
||||
{
|
||||
Id = "ev-001",
|
||||
Type = "sbom",
|
||||
Summary = "Component lodash@4.17.20 found in SBOM",
|
||||
Content = "Package: lodash, Version: 4.17.20, License: MIT",
|
||||
Source = "sbom-scan",
|
||||
Confidence = 0.99,
|
||||
CollectedAt = "2024-01-15T10:00:00Z"
|
||||
}
|
||||
],
|
||||
ReachabilityEvidence =
|
||||
[
|
||||
new EvidenceNode
|
||||
{
|
||||
Id = "ev-002",
|
||||
Type = "reachability",
|
||||
Summary = "Vulnerable function is reachable",
|
||||
Content = "Call path: main.js -> utils.js -> lodash.merge()",
|
||||
Source = "static-analysis",
|
||||
Confidence = 0.85,
|
||||
CollectedAt = "2024-01-15T10:05:00Z"
|
||||
}
|
||||
],
|
||||
RuntimeEvidence = [],
|
||||
VexEvidence =
|
||||
[
|
||||
new EvidenceNode
|
||||
{
|
||||
Id = "ev-003",
|
||||
Type = "vex",
|
||||
Summary = "No vendor VEX statement",
|
||||
Content = "No applicable VEX statements found",
|
||||
Source = "vex-lookup",
|
||||
Confidence = 0.5,
|
||||
CollectedAt = "2024-01-15T10:10:00Z"
|
||||
}
|
||||
],
|
||||
PatchEvidence = [],
|
||||
ContextHash = ComputeHash("full-evidence-context")
|
||||
};
|
||||
|
||||
private static EvidenceContext CreateMinimalEvidenceContext() => new()
|
||||
{
|
||||
SbomEvidence =
|
||||
[
|
||||
new EvidenceNode
|
||||
{
|
||||
Id = "ev-min-001",
|
||||
Type = "sbom",
|
||||
Summary = "Component found",
|
||||
Content = "Package exists",
|
||||
Source = "sbom",
|
||||
Confidence = 0.7,
|
||||
CollectedAt = "2024-01-15T10:00:00Z"
|
||||
}
|
||||
],
|
||||
ReachabilityEvidence = [],
|
||||
RuntimeEvidence = [],
|
||||
VexEvidence = [],
|
||||
PatchEvidence = [],
|
||||
ContextHash = ComputeHash("minimal-evidence-context")
|
||||
};
|
||||
|
||||
private static string ComputeHash(string content)
|
||||
{
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
|
||||
return Convert.ToHexStringLower(bytes);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Stub Implementations
|
||||
|
||||
private sealed class StubEvidenceRetrievalService : IEvidenceRetrievalService
|
||||
{
|
||||
private readonly EvidenceContext _context;
|
||||
private readonly bool _validateResult;
|
||||
|
||||
public StubEvidenceRetrievalService(EvidenceContext context, bool validateResult = true)
|
||||
{
|
||||
_context = context;
|
||||
_validateResult = validateResult;
|
||||
}
|
||||
|
||||
public Task<EvidenceContext> RetrieveEvidenceAsync(
|
||||
string findingId, string artifactDigest, string vulnerabilityId,
|
||||
string? componentPurl = null, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_context);
|
||||
|
||||
public Task<EvidenceNode?> GetEvidenceNodeAsync(string evidenceId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_context.AllEvidence.FirstOrDefault(e => e.Id == evidenceId));
|
||||
|
||||
public Task<bool> ValidateEvidenceAsync(IEnumerable<string> evidenceIds, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_validateResult);
|
||||
}
|
||||
|
||||
private sealed class StubExplanationPromptService : IExplanationPromptService
|
||||
{
|
||||
private readonly string _templateVersion;
|
||||
|
||||
public StubExplanationPromptService(string templateVersion = "explain-v1.0")
|
||||
{
|
||||
_templateVersion = templateVersion;
|
||||
}
|
||||
|
||||
public Task<ExplanationPrompt> BuildPromptAsync(
|
||||
ExplanationRequest request, EvidenceContext evidence, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new ExplanationPrompt
|
||||
{
|
||||
Content = $"Explain {request.VulnerabilityId} for {request.ExplanationType}",
|
||||
TemplateVersion = _templateVersion
|
||||
});
|
||||
|
||||
public Task<ExplanationSummary> GenerateSummaryAsync(
|
||||
string content, ExplanationType type, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new ExplanationSummary
|
||||
{
|
||||
Line1 = "What: Vulnerability detected",
|
||||
Line2 = "Why: Reachable code path",
|
||||
Line3 = "Action: Update dependency"
|
||||
});
|
||||
}
|
||||
|
||||
private sealed class StubExplanationInferenceClient : IExplanationInferenceClient
|
||||
{
|
||||
private readonly string _content;
|
||||
private readonly double _confidence;
|
||||
private readonly string _modelId;
|
||||
|
||||
public StubExplanationInferenceClient(string content, double confidence, string modelId = "stub-model:v1")
|
||||
{
|
||||
_content = content;
|
||||
_confidence = confidence;
|
||||
_modelId = modelId;
|
||||
}
|
||||
|
||||
public Task<ExplanationInferenceResult> GenerateAsync(
|
||||
ExplanationPrompt prompt, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new ExplanationInferenceResult
|
||||
{
|
||||
Content = _content,
|
||||
Confidence = _confidence,
|
||||
ModelId = _modelId
|
||||
});
|
||||
}
|
||||
|
||||
private sealed class StubCitationExtractor : ICitationExtractor
|
||||
{
|
||||
private readonly double _verifiedRate;
|
||||
|
||||
public StubCitationExtractor(double verifiedRate)
|
||||
{
|
||||
_verifiedRate = verifiedRate;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<ExplanationCitation>> ExtractCitationsAsync(
|
||||
string content, EvidenceContext evidence, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var citations = new List<ExplanationCitation>();
|
||||
var evidenceList = evidence.AllEvidence.ToList();
|
||||
|
||||
for (int i = 0; i < evidenceList.Count; i++)
|
||||
{
|
||||
var ev = evidenceList[i];
|
||||
citations.Add(new ExplanationCitation
|
||||
{
|
||||
ClaimText = $"Claim about {ev.Type}",
|
||||
EvidenceId = ev.Id,
|
||||
EvidenceType = ev.Type,
|
||||
Verified = i < (int)(evidenceList.Count * _verifiedRate),
|
||||
EvidenceExcerpt = ev.Summary
|
||||
});
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<ExplanationCitation>>(citations);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryExplanationStore : IExplanationStore
|
||||
{
|
||||
private readonly Dictionary<string, ExplanationResult> _results = new();
|
||||
private readonly Dictionary<string, ExplanationRequest> _requests = new();
|
||||
|
||||
public Task StoreAsync(ExplanationResult result, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_results[result.ExplanationId] = result;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<ExplanationResult?> GetAsync(string explanationId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_results.GetValueOrDefault(explanationId));
|
||||
|
||||
public Task<ExplanationRequest?> GetRequestAsync(string explanationId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_requests.GetValueOrDefault(explanationId));
|
||||
|
||||
public void StoreRequest(string explanationId, ExplanationRequest request)
|
||||
=> _requests[explanationId] = request;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,500 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using StellaOps.AdvisoryAI.Explanation;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Golden tests for deterministic explanation replay.
|
||||
/// Verifies that replaying an explanation with the same inputs produces identical output.
|
||||
/// Sprint: SPRINT_20251226_015_AI_zastava_companion
|
||||
/// Task: ZASTAVA-20
|
||||
/// </summary>
|
||||
public sealed class ExplanationReplayGoldenTests
|
||||
{
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ReplayAsync_WithSameInputs_ProducesIdenticalOutput()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
|
||||
// Act - Generate original
|
||||
var original = await generator.GenerateAsync(request);
|
||||
store.StoreRequest(original.ExplanationId, request);
|
||||
|
||||
// Act - Replay
|
||||
var replayed = await generator.ReplayAsync(original.ExplanationId);
|
||||
|
||||
// Assert - Output should be identical
|
||||
replayed.OutputHash.Should().Be(original.OutputHash);
|
||||
replayed.Content.Should().Be(original.Content);
|
||||
replayed.InputHashes.Should().BeEquivalentTo(original.InputHashes);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ReplayAsync_PreservesExplanationStructure()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
var original = await generator.GenerateAsync(request);
|
||||
store.StoreRequest(original.ExplanationId, request);
|
||||
|
||||
// Act
|
||||
var replayed = await generator.ReplayAsync(original.ExplanationId);
|
||||
|
||||
// Assert
|
||||
replayed.Citations.Count.Should().Be(original.Citations.Count);
|
||||
replayed.EvidenceRefs.Should().BeEquivalentTo(original.EvidenceRefs);
|
||||
replayed.ConfidenceScore.Should().Be(original.ConfidenceScore);
|
||||
replayed.CitationRate.Should().Be(original.CitationRate);
|
||||
replayed.Authority.Should().Be(original.Authority);
|
||||
replayed.ModelId.Should().Be(original.ModelId);
|
||||
replayed.PromptTemplateVersion.Should().Be(original.PromptTemplateVersion);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ReplayAsync_WithChangedEvidence_ThrowsException()
|
||||
{
|
||||
// Arrange
|
||||
var originalContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateGeneratorWithChangingEvidence(store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
var original = await generator.GenerateAsync(request);
|
||||
store.StoreRequest(original.ExplanationId, request);
|
||||
|
||||
// Mark evidence as changed
|
||||
generator.MarkEvidenceAsChanged();
|
||||
|
||||
// Act & Assert
|
||||
var act = async () => await generator.ReplayAsync(original.ExplanationId);
|
||||
await act.Should().ThrowAsync<InvalidOperationException>()
|
||||
.WithMessage("*evidence has changed*");
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task MultipleReplays_ProduceIdenticalResults()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
var original = await generator.GenerateAsync(request);
|
||||
store.StoreRequest(original.ExplanationId, request);
|
||||
|
||||
// Act - Replay multiple times
|
||||
var replay1 = await generator.ReplayAsync(original.ExplanationId);
|
||||
var replay2 = await generator.ReplayAsync(original.ExplanationId);
|
||||
var replay3 = await generator.ReplayAsync(original.ExplanationId);
|
||||
|
||||
// Assert - All should be identical
|
||||
replay1.OutputHash.Should().Be(original.OutputHash);
|
||||
replay2.OutputHash.Should().Be(original.OutputHash);
|
||||
replay3.OutputHash.Should().Be(original.OutputHash);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task InputHashOrder_IsConsistent()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
|
||||
// Act - Generate twice
|
||||
var result1 = await generator.GenerateAsync(request);
|
||||
var result2 = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert - Input hashes should be in same order
|
||||
result1.InputHashes.Should().HaveCount(3);
|
||||
result2.InputHashes.Should().HaveCount(3);
|
||||
for (int i = 0; i < result1.InputHashes.Count; i++)
|
||||
{
|
||||
result1.InputHashes[i].Should().Be(result2.InputHashes[i],
|
||||
$"Input hash at index {i} should be identical");
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ExplanationId_IsDeterministicFromInputsAndOutput()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
|
||||
// Act
|
||||
var result1 = await generator.GenerateAsync(request);
|
||||
var result2 = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert - Same inputs + same output = same ID
|
||||
result1.ExplanationId.Should().Be(result2.ExplanationId);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task DifferentInputs_ProduceDifferentIds()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request1 = CreateDeterministicRequest() with { VulnerabilityId = "CVE-2024-0001" };
|
||||
var request2 = CreateDeterministicRequest() with { VulnerabilityId = "CVE-2024-0002" };
|
||||
|
||||
// Act
|
||||
var result1 = await generator.GenerateAsync(request1);
|
||||
var result2 = await generator.GenerateAsync(request2);
|
||||
|
||||
// Assert
|
||||
result1.ExplanationId.Should().NotBe(result2.ExplanationId);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GoldenOutput_MatchesExpectedFormat()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert - Verify golden format
|
||||
result.ExplanationId.Should().StartWith("sha256:");
|
||||
result.OutputHash.Should().HaveLength(64); // SHA-256 hex
|
||||
result.GeneratedAt.Should().MatchRegex(@"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}");
|
||||
result.InputHashes.Should().AllSatisfy(h => h.Length.Should().Be(64));
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CitationVerification_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
|
||||
// Act
|
||||
var result1 = await generator.GenerateAsync(request);
|
||||
var result2 = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert - Citations should be identical in order and verification status
|
||||
result1.Citations.Count.Should().Be(result2.Citations.Count);
|
||||
for (int i = 0; i < result1.Citations.Count; i++)
|
||||
{
|
||||
result1.Citations[i].ClaimText.Should().Be(result2.Citations[i].ClaimText);
|
||||
result1.Citations[i].EvidenceId.Should().Be(result2.Citations[i].EvidenceId);
|
||||
result1.Citations[i].Verified.Should().Be(result2.Citations[i].Verified);
|
||||
}
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SummaryGeneration_IsDeterministic()
|
||||
{
|
||||
// Arrange
|
||||
var evidenceContext = CreateDeterministicEvidenceContext();
|
||||
var store = new InMemoryExplanationStoreWithRequests();
|
||||
var generator = CreateDeterministicGenerator(evidenceContext, store);
|
||||
|
||||
var request = CreateDeterministicRequest();
|
||||
|
||||
// Act
|
||||
var result1 = await generator.GenerateAsync(request);
|
||||
var result2 = await generator.GenerateAsync(request);
|
||||
|
||||
// Assert
|
||||
result1.Summary.Line1.Should().Be(result2.Summary.Line1);
|
||||
result1.Summary.Line2.Should().Be(result2.Summary.Line2);
|
||||
result1.Summary.Line3.Should().Be(result2.Summary.Line3);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void OutputHash_MatchesContentHash()
|
||||
{
|
||||
// Arrange
|
||||
var content = "This is deterministic explanation content.";
|
||||
var expectedHash = ComputeHash(content);
|
||||
|
||||
// Act
|
||||
var actualHash = ComputeHash(content);
|
||||
|
||||
// Assert
|
||||
actualHash.Should().Be(expectedHash);
|
||||
actualHash.Should().HaveLength(64);
|
||||
}
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static ExplanationRequest CreateDeterministicRequest() => new()
|
||||
{
|
||||
FindingId = "golden-finding-001",
|
||||
ArtifactDigest = "sha256:golden123abc",
|
||||
Scope = "image",
|
||||
ScopeId = "golden-image:v1.0.0",
|
||||
ExplanationType = ExplanationType.Full,
|
||||
VulnerabilityId = "CVE-2024-GOLDEN",
|
||||
ComponentPurl = "pkg:npm/golden-pkg@1.0.0",
|
||||
PlainLanguage = false,
|
||||
MaxLength = 0,
|
||||
CorrelationId = "golden-corr-001"
|
||||
};
|
||||
|
||||
private static EvidenceContext CreateDeterministicEvidenceContext() => new()
|
||||
{
|
||||
SbomEvidence =
|
||||
[
|
||||
new EvidenceNode
|
||||
{
|
||||
Id = "golden-ev-001",
|
||||
Type = "sbom",
|
||||
Summary = "Golden component found",
|
||||
Content = "Package: golden-pkg, Version: 1.0.0",
|
||||
Source = "golden-sbom",
|
||||
Confidence = 0.99,
|
||||
CollectedAt = "2024-01-01T00:00:00Z"
|
||||
}
|
||||
],
|
||||
ReachabilityEvidence =
|
||||
[
|
||||
new EvidenceNode
|
||||
{
|
||||
Id = "golden-ev-002",
|
||||
Type = "reachability",
|
||||
Summary = "Golden function reachable",
|
||||
Content = "Call path: entry -> golden_func()",
|
||||
Source = "golden-analysis",
|
||||
Confidence = 0.95,
|
||||
CollectedAt = "2024-01-01T00:00:01Z"
|
||||
}
|
||||
],
|
||||
RuntimeEvidence = [],
|
||||
VexEvidence = [],
|
||||
PatchEvidence = [],
|
||||
ContextHash = ComputeHash("golden-evidence-context-v1")
|
||||
};
|
||||
|
||||
private static EvidenceAnchoredExplanationGenerator CreateDeterministicGenerator(
|
||||
EvidenceContext evidenceContext,
|
||||
InMemoryExplanationStoreWithRequests store)
|
||||
{
|
||||
var evidenceService = new DeterministicEvidenceService(evidenceContext);
|
||||
var promptService = new DeterministicPromptService();
|
||||
var inferenceClient = new DeterministicInferenceClient();
|
||||
var citationExtractor = new DeterministicCitationExtractor();
|
||||
|
||||
return new EvidenceAnchoredExplanationGenerator(
|
||||
evidenceService, promptService, inferenceClient, citationExtractor, store);
|
||||
}
|
||||
|
||||
private static ChangingEvidenceGenerator CreateGeneratorWithChangingEvidence(
|
||||
InMemoryExplanationStoreWithRequests store)
|
||||
{
|
||||
return new ChangingEvidenceGenerator(store);
|
||||
}
|
||||
|
||||
private static string ComputeHash(string content)
|
||||
{
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
|
||||
return Convert.ToHexStringLower(bytes);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Deterministic Test Doubles
|
||||
|
||||
private sealed class DeterministicEvidenceService : IEvidenceRetrievalService
|
||||
{
|
||||
private readonly EvidenceContext _context;
|
||||
|
||||
public DeterministicEvidenceService(EvidenceContext context) => _context = context;
|
||||
|
||||
public Task<EvidenceContext> RetrieveEvidenceAsync(
|
||||
string findingId, string artifactDigest, string vulnerabilityId,
|
||||
string? componentPurl = null, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_context);
|
||||
|
||||
public Task<EvidenceNode?> GetEvidenceNodeAsync(string evidenceId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_context.AllEvidence.FirstOrDefault(e => e.Id == evidenceId));
|
||||
|
||||
public Task<bool> ValidateEvidenceAsync(IEnumerable<string> evidenceIds, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(true);
|
||||
}
|
||||
|
||||
private sealed class DeterministicPromptService : IExplanationPromptService
|
||||
{
|
||||
public Task<ExplanationPrompt> BuildPromptAsync(
|
||||
ExplanationRequest request, EvidenceContext evidence, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new ExplanationPrompt
|
||||
{
|
||||
Content = $"GOLDEN_PROMPT:{request.VulnerabilityId}:{evidence.ContextHash}",
|
||||
TemplateVersion = "golden-template-v1.0"
|
||||
});
|
||||
|
||||
public Task<ExplanationSummary> GenerateSummaryAsync(
|
||||
string content, ExplanationType type, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(new ExplanationSummary
|
||||
{
|
||||
Line1 = "Golden: What happened",
|
||||
Line2 = "Golden: Why it matters",
|
||||
Line3 = "Golden: Next steps"
|
||||
});
|
||||
}
|
||||
|
||||
private sealed class DeterministicInferenceClient : IExplanationInferenceClient
|
||||
{
|
||||
public Task<ExplanationInferenceResult> GenerateAsync(
|
||||
ExplanationPrompt prompt, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Deterministic output based on prompt hash
|
||||
var content = $"GOLDEN_EXPLANATION:hash={ComputeHash(prompt.Content)}";
|
||||
return Task.FromResult(new ExplanationInferenceResult
|
||||
{
|
||||
Content = content,
|
||||
Confidence = 0.95,
|
||||
ModelId = "golden-model:v1.0"
|
||||
});
|
||||
}
|
||||
|
||||
private static string ComputeHash(string content)
|
||||
{
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
|
||||
return Convert.ToHexStringLower(bytes)[..16];
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class DeterministicCitationExtractor : ICitationExtractor
|
||||
{
|
||||
public Task<IReadOnlyList<ExplanationCitation>> ExtractCitationsAsync(
|
||||
string content, EvidenceContext evidence, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Deterministic citations based on evidence order
|
||||
var citations = evidence.AllEvidence.Select((ev, i) => new ExplanationCitation
|
||||
{
|
||||
ClaimText = $"Golden claim {i + 1}",
|
||||
EvidenceId = ev.Id,
|
||||
EvidenceType = ev.Type,
|
||||
Verified = true,
|
||||
EvidenceExcerpt = ev.Summary
|
||||
}).ToList();
|
||||
|
||||
return Task.FromResult<IReadOnlyList<ExplanationCitation>>(citations);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class InMemoryExplanationStoreWithRequests : IExplanationStore
|
||||
{
|
||||
private readonly Dictionary<string, ExplanationResult> _results = new();
|
||||
private readonly Dictionary<string, ExplanationRequest> _requests = new();
|
||||
|
||||
public Task StoreAsync(ExplanationResult result, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_results[result.ExplanationId] = result;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<ExplanationResult?> GetAsync(string explanationId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_results.GetValueOrDefault(explanationId));
|
||||
|
||||
public Task<ExplanationRequest?> GetRequestAsync(string explanationId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(_requests.GetValueOrDefault(explanationId));
|
||||
|
||||
public void StoreRequest(string explanationId, ExplanationRequest request)
|
||||
=> _requests[explanationId] = request;
|
||||
}
|
||||
|
||||
private sealed class ChangingEvidenceGenerator : IExplanationGenerator
|
||||
{
|
||||
private readonly InMemoryExplanationStoreWithRequests _store;
|
||||
private bool _evidenceChanged = false;
|
||||
|
||||
public ChangingEvidenceGenerator(InMemoryExplanationStoreWithRequests store)
|
||||
{
|
||||
_store = store;
|
||||
}
|
||||
|
||||
public void MarkEvidenceAsChanged() => _evidenceChanged = true;
|
||||
|
||||
public async Task<ExplanationResult> GenerateAsync(ExplanationRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var result = new ExplanationResult
|
||||
{
|
||||
ExplanationId = $"sha256:{ComputeHash(JsonSerializer.Serialize(request))}",
|
||||
Content = "Test content",
|
||||
Summary = new ExplanationSummary { Line1 = "L1", Line2 = "L2", Line3 = "L3" },
|
||||
Citations = [],
|
||||
ConfidenceScore = 0.9,
|
||||
CitationRate = 0.9,
|
||||
Authority = ExplanationAuthority.EvidenceBacked,
|
||||
EvidenceRefs = ["ev-001"],
|
||||
ModelId = "test-model",
|
||||
PromptTemplateVersion = "v1",
|
||||
InputHashes = [ComputeHash("input")],
|
||||
GeneratedAt = DateTime.UtcNow.ToString("O"),
|
||||
OutputHash = ComputeHash("Test content")
|
||||
};
|
||||
|
||||
await _store.StoreAsync(result, cancellationToken);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<ExplanationResult> ReplayAsync(string explanationId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var original = await _store.GetAsync(explanationId, cancellationToken)
|
||||
?? throw new InvalidOperationException($"Explanation {explanationId} not found");
|
||||
|
||||
if (_evidenceChanged)
|
||||
{
|
||||
throw new InvalidOperationException("Input evidence has changed since original explanation");
|
||||
}
|
||||
|
||||
return original;
|
||||
}
|
||||
|
||||
public Task<bool> ValidateAsync(ExplanationResult result, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(!_evidenceChanged);
|
||||
|
||||
private static string ComputeHash(string content)
|
||||
{
|
||||
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
|
||||
return Convert.ToHexStringLower(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -19,13 +19,15 @@ using StellaOps.AdvisoryAI.Tools;
|
||||
using StellaOps.AdvisoryAI.Tests.TestUtilities;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class FileSystemAdvisoryOutputStoreTests : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _temp = TempDirectory.Create();
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SaveAndRetrieve_RoundTripsOutput()
|
||||
{
|
||||
var store = CreateStore();
|
||||
@@ -41,7 +43,8 @@ public sealed class FileSystemAdvisoryOutputStoreTests : IDisposable
|
||||
retrieved.Metadata["inference.model_id"].Should().Be("local.prompt-preview");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task TryGetAsync_ReturnsNullWhenFileMissing()
|
||||
{
|
||||
var store = CreateStore();
|
||||
|
||||
@@ -19,13 +19,15 @@ using StellaOps.AdvisoryAI.Prompting;
|
||||
using StellaOps.AdvisoryAI.Tools;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class FileSystemAdvisoryPersistenceTests : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _tempDir = new();
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task PlanCache_PersistsPlanOnDisk()
|
||||
{
|
||||
var serviceOptions = Options.Create(new AdvisoryAiServiceOptions
|
||||
@@ -54,7 +56,8 @@ public sealed class FileSystemAdvisoryPersistenceTests : IDisposable
|
||||
reloaded.Metadata.Should().ContainKey("advisory_key").WhoseValue.Should().Be("adv-key");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task OutputStore_PersistsOutputOnDisk()
|
||||
{
|
||||
var serviceOptions = Options.Create(new AdvisoryAiServiceOptions
|
||||
|
||||
@@ -16,13 +16,15 @@ using StellaOps.AdvisoryAI.Tools;
|
||||
using StellaOps.AdvisoryAI.Tests.TestUtilities;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class FileSystemAdvisoryPlanCacheTests : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _temp = TempDirectory.Create();
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SetAndRetrieve_RoundTripsPlan()
|
||||
{
|
||||
var cache = CreateCache();
|
||||
@@ -36,7 +38,8 @@ public sealed class FileSystemAdvisoryPlanCacheTests : IDisposable
|
||||
retrieved.Metadata.Should().ContainKey("task_type");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task TryGetAsync_WhenExpired_ReturnsNull()
|
||||
{
|
||||
var clock = new DeterministicTimeProvider(new DateTimeOffset(2025, 11, 9, 0, 0, 0, TimeSpan.Zero));
|
||||
@@ -50,7 +53,8 @@ public sealed class FileSystemAdvisoryPlanCacheTests : IDisposable
|
||||
retrieved.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task BulkSeedAsync_RemainsDeterministicAcrossInstances()
|
||||
{
|
||||
var clock = new DeterministicTimeProvider(new DateTimeOffset(2025, 11, 9, 0, 0, 0, TimeSpan.Zero));
|
||||
|
||||
@@ -8,6 +8,7 @@ using StellaOps.AdvisoryAI.Orchestration;
|
||||
using StellaOps.AdvisoryAI.Queue;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class FileSystemAdvisoryTaskQueueTests : IDisposable
|
||||
@@ -20,7 +21,8 @@ public sealed class FileSystemAdvisoryTaskQueueTests : IDisposable
|
||||
Directory.CreateDirectory(_root);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task EnqueueAndDequeue_RoundTripsMessage()
|
||||
{
|
||||
var options = Options.Create(new AdvisoryAiServiceOptions
|
||||
|
||||
@@ -0,0 +1,795 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.AdvisoryAI.Inference;
|
||||
using StellaOps.AdvisoryAI.Inference.LlmProviders;
|
||||
using Xunit;
|
||||
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for offline AI inference infrastructure.
|
||||
/// Sprint: SPRINT_20251226_019_AI_offline_inference
|
||||
/// Task: OFFLINE-25
|
||||
/// </summary>
|
||||
public sealed class OfflineInferenceIntegrationTests : IDisposable
|
||||
{
|
||||
private readonly string _tempPath;
|
||||
private readonly InMemoryLlmInferenceCache _cache;
|
||||
private readonly StubLlmProvider _stubProvider;
|
||||
|
||||
public OfflineInferenceIntegrationTests()
|
||||
{
|
||||
_tempPath = Path.Combine(Path.GetTempPath(), $"stellaops_offline_tests_{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_tempPath);
|
||||
|
||||
var cacheOptions = Options.Create(new LlmInferenceCacheOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DeterministicOnly = true,
|
||||
DefaultTtl = TimeSpan.FromDays(7)
|
||||
});
|
||||
|
||||
_cache = new InMemoryLlmInferenceCache(
|
||||
cacheOptions,
|
||||
NullLogger<InMemoryLlmInferenceCache>.Instance);
|
||||
|
||||
_stubProvider = new StubLlmProvider();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cache.Dispose();
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_tempPath))
|
||||
{
|
||||
Directory.Delete(_tempPath, recursive: true);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore cleanup errors
|
||||
}
|
||||
}
|
||||
|
||||
#region Local Inference Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CompleteAsync_WithDeterministicSettings_ReturnsDeterministicResult()
|
||||
{
|
||||
// Arrange
|
||||
var request = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Analyze CVE-2024-1234 for log4j",
|
||||
SystemPrompt = "You are a security analyst.",
|
||||
Temperature = 0,
|
||||
Seed = 42,
|
||||
MaxTokens = 1024
|
||||
};
|
||||
|
||||
// Act
|
||||
var result1 = await _stubProvider.CompleteAsync(request);
|
||||
var result2 = await _stubProvider.CompleteAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.True(result1.Deterministic);
|
||||
Assert.True(result2.Deterministic);
|
||||
Assert.Equal(result1.Content, result2.Content);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CompleteAsync_WithProviderAvailabilityCheck_ReturnsTrue()
|
||||
{
|
||||
// Act
|
||||
var isAvailable = await _stubProvider.IsAvailableAsync();
|
||||
|
||||
// Assert
|
||||
Assert.True(isAvailable);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CompleteStreamAsync_YieldsChunks()
|
||||
{
|
||||
// Arrange
|
||||
var request = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Test streaming",
|
||||
Temperature = 0
|
||||
};
|
||||
|
||||
// Act
|
||||
var chunks = new List<LlmStreamChunk>();
|
||||
await foreach (var chunk in _stubProvider.CompleteStreamAsync(request))
|
||||
{
|
||||
chunks.Add(chunk);
|
||||
}
|
||||
|
||||
// Assert
|
||||
Assert.NotEmpty(chunks);
|
||||
Assert.Contains(chunks, c => c.IsFinal);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Inference Cache Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task Cache_DeterministicRequest_CachesResult()
|
||||
{
|
||||
// Arrange
|
||||
var request = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Cached prompt",
|
||||
Temperature = 0,
|
||||
Seed = 42
|
||||
};
|
||||
|
||||
var result = new LlmCompletionResult
|
||||
{
|
||||
Content = "Cached response",
|
||||
ModelId = "test-model",
|
||||
ProviderId = "stub",
|
||||
Deterministic = true,
|
||||
OutputTokens = 10
|
||||
};
|
||||
|
||||
// Act
|
||||
await _cache.SetAsync(request, "stub", result);
|
||||
var cached = await _cache.TryGetAsync(request, "stub");
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(cached);
|
||||
Assert.Equal(result.Content, cached.Content);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task Cache_NonDeterministicRequest_DoesNotCache()
|
||||
{
|
||||
// Arrange
|
||||
var options = Options.Create(new LlmInferenceCacheOptions
|
||||
{
|
||||
Enabled = true,
|
||||
DeterministicOnly = true
|
||||
});
|
||||
|
||||
using var cache = new InMemoryLlmInferenceCache(
|
||||
options, NullLogger<InMemoryLlmInferenceCache>.Instance);
|
||||
|
||||
var request = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Non-deterministic",
|
||||
Temperature = 0.7 // Non-deterministic
|
||||
};
|
||||
|
||||
var result = new LlmCompletionResult
|
||||
{
|
||||
Content = "Response",
|
||||
ModelId = "test-model",
|
||||
ProviderId = "stub",
|
||||
Deterministic = false
|
||||
};
|
||||
|
||||
// Act
|
||||
await cache.SetAsync(request, "stub", result);
|
||||
var cached = await cache.TryGetAsync(request, "stub");
|
||||
|
||||
// Assert
|
||||
Assert.Null(cached);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task Cache_SameInputsDifferentSeeds_SeparateCacheEntries()
|
||||
{
|
||||
// Arrange
|
||||
var request1 = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Test prompt",
|
||||
Temperature = 0,
|
||||
Seed = 42
|
||||
};
|
||||
|
||||
var request2 = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Test prompt",
|
||||
Temperature = 0,
|
||||
Seed = 123
|
||||
};
|
||||
|
||||
var result1 = new LlmCompletionResult
|
||||
{
|
||||
Content = "Response with seed 42",
|
||||
ModelId = "test-model",
|
||||
ProviderId = "stub",
|
||||
Deterministic = true
|
||||
};
|
||||
|
||||
var result2 = new LlmCompletionResult
|
||||
{
|
||||
Content = "Response with seed 123",
|
||||
ModelId = "test-model",
|
||||
ProviderId = "stub",
|
||||
Deterministic = true
|
||||
};
|
||||
|
||||
// Act
|
||||
await _cache.SetAsync(request1, "stub", result1);
|
||||
await _cache.SetAsync(request2, "stub", result2);
|
||||
|
||||
var cached1 = await _cache.TryGetAsync(request1, "stub");
|
||||
var cached2 = await _cache.TryGetAsync(request2, "stub");
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(cached1);
|
||||
Assert.NotNull(cached2);
|
||||
Assert.NotEqual(cached1.Content, cached2.Content);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void Cache_Statistics_TracksHitsAndMisses()
|
||||
{
|
||||
// Act
|
||||
var stats = _cache.GetStatistics();
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(stats);
|
||||
Assert.Equal(0, stats.Hits);
|
||||
Assert.True(stats.HitRate >= 0 && stats.HitRate <= 1);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CachingLlmProvider_UsesCache()
|
||||
{
|
||||
// Arrange
|
||||
var countingProvider = new CallCountingLlmProvider();
|
||||
var cachingProvider = new CachingLlmProvider(
|
||||
countingProvider,
|
||||
_cache,
|
||||
NullLogger<CachingLlmProvider>.Instance);
|
||||
|
||||
var request = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Test caching",
|
||||
Temperature = 0,
|
||||
Seed = 42
|
||||
};
|
||||
|
||||
// Act - First call hits provider
|
||||
var result1 = await cachingProvider.CompleteAsync(request);
|
||||
Assert.Equal(1, countingProvider.CallCount);
|
||||
|
||||
// Act - Second call should use cache
|
||||
var result2 = await cachingProvider.CompleteAsync(request);
|
||||
Assert.Equal(1, countingProvider.CallCount); // Still 1, used cache
|
||||
|
||||
// Assert
|
||||
Assert.Equal(result1.Content, result2.Content);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Bundle Verification Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task BundleManager_VerifyBundle_ValidBundle_ReturnsValid()
|
||||
{
|
||||
// Arrange
|
||||
var bundlePath = Path.Combine(_tempPath, "valid-bundle");
|
||||
CreateValidBundle(bundlePath);
|
||||
|
||||
var manager = new FileSystemModelBundleManager(_tempPath);
|
||||
|
||||
// Act
|
||||
var result = await manager.VerifyBundleAsync(bundlePath);
|
||||
|
||||
// Assert
|
||||
Assert.True(result.Valid);
|
||||
Assert.Empty(result.FailedFiles);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task BundleManager_VerifyBundle_MissingManifest_ReturnsInvalid()
|
||||
{
|
||||
// Arrange
|
||||
var bundlePath = Path.Combine(_tempPath, "no-manifest");
|
||||
Directory.CreateDirectory(bundlePath);
|
||||
|
||||
var manager = new FileSystemModelBundleManager(_tempPath);
|
||||
|
||||
// Act
|
||||
var result = await manager.VerifyBundleAsync(bundlePath);
|
||||
|
||||
// Assert
|
||||
Assert.False(result.Valid);
|
||||
Assert.NotNull(result.ErrorMessage);
|
||||
Assert.Contains("manifest.json", result.ErrorMessage);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task BundleManager_VerifyBundle_CorruptedFile_ReturnsInvalid()
|
||||
{
|
||||
// Arrange
|
||||
var bundlePath = Path.Combine(_tempPath, "corrupted-bundle");
|
||||
CreateValidBundle(bundlePath);
|
||||
|
||||
// Corrupt a file
|
||||
var modelFile = Path.Combine(bundlePath, "model.gguf");
|
||||
await File.WriteAllTextAsync(modelFile, "corrupted data");
|
||||
|
||||
var manager = new FileSystemModelBundleManager(_tempPath);
|
||||
|
||||
// Act
|
||||
var result = await manager.VerifyBundleAsync(bundlePath);
|
||||
|
||||
// Assert
|
||||
Assert.False(result.Valid);
|
||||
Assert.NotEmpty(result.FailedFiles);
|
||||
Assert.Contains(result.FailedFiles, f => f.Contains("model.gguf"));
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task BundleManager_VerifyBundle_MissingFile_ReturnsInvalid()
|
||||
{
|
||||
// Arrange
|
||||
var bundlePath = Path.Combine(_tempPath, "missing-file-bundle");
|
||||
CreateValidBundle(bundlePath);
|
||||
|
||||
// Delete a file
|
||||
File.Delete(Path.Combine(bundlePath, "tokenizer.json"));
|
||||
|
||||
var manager = new FileSystemModelBundleManager(_tempPath);
|
||||
|
||||
// Act
|
||||
var result = await manager.VerifyBundleAsync(bundlePath);
|
||||
|
||||
// Assert
|
||||
Assert.False(result.Valid);
|
||||
Assert.NotEmpty(result.FailedFiles);
|
||||
Assert.Contains(result.FailedFiles, f => f.Contains("missing"));
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task BundleManager_ListBundles_ReturnsAvailableBundles()
|
||||
{
|
||||
// Arrange
|
||||
CreateValidBundle(Path.Combine(_tempPath, "bundle1"));
|
||||
CreateValidBundle(Path.Combine(_tempPath, "bundle2"));
|
||||
|
||||
var manager = new FileSystemModelBundleManager(_tempPath);
|
||||
|
||||
// Act
|
||||
var bundles = await manager.ListBundlesAsync();
|
||||
|
||||
// Assert
|
||||
Assert.Equal(2, bundles.Count);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task BundleManager_GetManifest_ExistingBundle_ReturnsManifest()
|
||||
{
|
||||
// Arrange
|
||||
CreateValidBundle(Path.Combine(_tempPath, "test-bundle"));
|
||||
var manager = new FileSystemModelBundleManager(_tempPath);
|
||||
|
||||
// Act
|
||||
var manifest = await manager.GetManifestAsync("test-bundle");
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(manifest);
|
||||
Assert.Equal("test-model", manifest.Name);
|
||||
Assert.Equal("Apache-2.0", manifest.License);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task BundleManager_GetManifest_NonExistentBundle_ReturnsNull()
|
||||
{
|
||||
// Arrange
|
||||
var manager = new FileSystemModelBundleManager(_tempPath);
|
||||
|
||||
// Act
|
||||
var manifest = await manager.GetManifestAsync("nonexistent");
|
||||
|
||||
// Assert
|
||||
Assert.Null(manifest);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Offline Replay Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task OfflineReplay_SameInputs_ProducesSameOutput()
|
||||
{
|
||||
// Arrange
|
||||
var request = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Analyze vulnerability impact",
|
||||
SystemPrompt = "You are a security expert.",
|
||||
Temperature = 0,
|
||||
Seed = 42,
|
||||
MaxTokens = 1024
|
||||
};
|
||||
|
||||
// Simulate first run
|
||||
var originalResult = await _stubProvider.CompleteAsync(request);
|
||||
await _cache.SetAsync(request, "stub", originalResult);
|
||||
|
||||
// Simulate replay (offline)
|
||||
var replayResult = await _cache.TryGetAsync(request, "stub");
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(replayResult);
|
||||
Assert.Equal(originalResult.Content, replayResult.Content);
|
||||
Assert.True(originalResult.Deterministic);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task OfflineReplay_DifferentInputs_DifferentOutput()
|
||||
{
|
||||
// Arrange
|
||||
var request1 = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Input A",
|
||||
Temperature = 0,
|
||||
Seed = 42
|
||||
};
|
||||
|
||||
var request2 = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "Input B",
|
||||
Temperature = 0,
|
||||
Seed = 42
|
||||
};
|
||||
|
||||
// Act
|
||||
var result1 = await _stubProvider.CompleteAsync(request1);
|
||||
var result2 = await _stubProvider.CompleteAsync(request2);
|
||||
|
||||
// Assert
|
||||
Assert.NotEqual(result1.Content, result2.Content);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task Cache_Invalidation_RemovesEntries()
|
||||
{
|
||||
// Arrange
|
||||
var request = new LlmCompletionRequest
|
||||
{
|
||||
UserPrompt = "To be invalidated",
|
||||
Temperature = 0,
|
||||
Seed = 42
|
||||
};
|
||||
|
||||
var result = new LlmCompletionResult
|
||||
{
|
||||
Content = "Cached content",
|
||||
ModelId = "test-model",
|
||||
ProviderId = "stub",
|
||||
Deterministic = true
|
||||
};
|
||||
|
||||
await _cache.SetAsync(request, "stub", result);
|
||||
|
||||
// Verify it's cached
|
||||
var cached = await _cache.TryGetAsync(request, "stub");
|
||||
Assert.NotNull(cached);
|
||||
|
||||
// Act - Invalidate
|
||||
await _cache.InvalidateAsync("stub");
|
||||
|
||||
// Assert
|
||||
var afterInvalidation = await _cache.TryGetAsync(request, "stub");
|
||||
Assert.Null(afterInvalidation);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region LocalLlmConfig Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void LocalLlmConfig_DefaultValues_AreCorrect()
|
||||
{
|
||||
// Arrange & Act
|
||||
var config = new LocalLlmConfig
|
||||
{
|
||||
ModelPath = "/models/test.gguf",
|
||||
WeightsDigest = "abc123"
|
||||
};
|
||||
|
||||
// Assert
|
||||
Assert.Equal(ModelQuantization.Q4_K_M, config.Quantization);
|
||||
Assert.Equal(4096, config.ContextLength);
|
||||
Assert.Equal(InferenceDevice.Auto, config.Device);
|
||||
Assert.Equal(0, config.Temperature);
|
||||
Assert.Equal(42, config.Seed);
|
||||
Assert.True(config.FlashAttention);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void LocalLlmConfig_CustomValues_AreApplied()
|
||||
{
|
||||
// Arrange & Act
|
||||
var config = new LocalLlmConfig
|
||||
{
|
||||
ModelPath = "/models/llama3-8b.gguf",
|
||||
WeightsDigest = "sha256:abc123def456",
|
||||
Quantization = ModelQuantization.FP16,
|
||||
ContextLength = 8192,
|
||||
Device = InferenceDevice.CUDA,
|
||||
GpuLayers = 32,
|
||||
Threads = 8,
|
||||
Temperature = 0,
|
||||
Seed = 12345,
|
||||
FlashAttention = false,
|
||||
MaxTokens = 4096
|
||||
};
|
||||
|
||||
// Assert
|
||||
Assert.Equal("/models/llama3-8b.gguf", config.ModelPath);
|
||||
Assert.Equal(ModelQuantization.FP16, config.Quantization);
|
||||
Assert.Equal(8192, config.ContextLength);
|
||||
Assert.Equal(InferenceDevice.CUDA, config.Device);
|
||||
Assert.Equal(32, config.GpuLayers);
|
||||
Assert.Equal(12345, config.Seed);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Fallback Provider Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task FallbackLlmProvider_FirstAvailable_UsesFirstProvider()
|
||||
{
|
||||
// Arrange
|
||||
var factory = new StubLlmProviderFactory(new Dictionary<string, ILlmProvider>
|
||||
{
|
||||
["primary"] = new StubLlmProvider { IsAvailableResult = true, ProviderIdOverride = "primary" },
|
||||
["fallback"] = new StubLlmProvider { IsAvailableResult = true, ProviderIdOverride = "fallback" }
|
||||
});
|
||||
|
||||
var fallbackProvider = new FallbackLlmProvider(
|
||||
factory,
|
||||
new[] { "primary", "fallback" },
|
||||
NullLogger<FallbackLlmProvider>.Instance);
|
||||
|
||||
var request = new LlmCompletionRequest { UserPrompt = "Test" };
|
||||
|
||||
// Act
|
||||
var result = await fallbackProvider.CompleteAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Equal("primary", result.ProviderId);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task FallbackLlmProvider_FirstUnavailable_UsesFallback()
|
||||
{
|
||||
// Arrange
|
||||
var factory = new StubLlmProviderFactory(new Dictionary<string, ILlmProvider>
|
||||
{
|
||||
["primary"] = new StubLlmProvider { IsAvailableResult = false, ProviderIdOverride = "primary" },
|
||||
["fallback"] = new StubLlmProvider { IsAvailableResult = true, ProviderIdOverride = "fallback" }
|
||||
});
|
||||
|
||||
var fallbackProvider = new FallbackLlmProvider(
|
||||
factory,
|
||||
new[] { "primary", "fallback" },
|
||||
NullLogger<FallbackLlmProvider>.Instance);
|
||||
|
||||
var request = new LlmCompletionRequest { UserPrompt = "Test" };
|
||||
|
||||
// Act
|
||||
var result = await fallbackProvider.CompleteAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Equal("fallback", result.ProviderId);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task FallbackLlmProvider_AllUnavailable_ThrowsException()
|
||||
{
|
||||
// Arrange
|
||||
var factory = new StubLlmProviderFactory(new Dictionary<string, ILlmProvider>
|
||||
{
|
||||
["primary"] = new StubLlmProvider { IsAvailableResult = false },
|
||||
["fallback"] = new StubLlmProvider { IsAvailableResult = false }
|
||||
});
|
||||
|
||||
var fallbackProvider = new FallbackLlmProvider(
|
||||
factory,
|
||||
new[] { "primary", "fallback" },
|
||||
NullLogger<FallbackLlmProvider>.Instance);
|
||||
|
||||
var request = new LlmCompletionRequest { UserPrompt = "Test" };
|
||||
|
||||
// Act & Assert
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(() =>
|
||||
fallbackProvider.CompleteAsync(request));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private void CreateValidBundle(string bundlePath)
|
||||
{
|
||||
Directory.CreateDirectory(bundlePath);
|
||||
|
||||
// Create model file
|
||||
var modelContent = "fake model weights for testing";
|
||||
var modelPath = Path.Combine(bundlePath, "model.gguf");
|
||||
File.WriteAllText(modelPath, modelContent);
|
||||
|
||||
// Create tokenizer file
|
||||
var tokenizerContent = "{\"vocab_size\": 32000}";
|
||||
var tokenizerPath = Path.Combine(bundlePath, "tokenizer.json");
|
||||
File.WriteAllText(tokenizerPath, tokenizerContent);
|
||||
|
||||
// Compute digests
|
||||
using var sha256 = SHA256.Create();
|
||||
var modelDigest = Convert.ToHexStringLower(sha256.ComputeHash(Encoding.UTF8.GetBytes(modelContent)));
|
||||
var tokenizerDigest = Convert.ToHexStringLower(sha256.ComputeHash(Encoding.UTF8.GetBytes(tokenizerContent)));
|
||||
|
||||
// Create manifest
|
||||
var manifest = new ModelBundleManifest
|
||||
{
|
||||
Name = "test-model",
|
||||
License = "Apache-2.0",
|
||||
SizeCategory = "7B",
|
||||
Quantizations = new[] { "Q4_K_M", "FP16" },
|
||||
CreatedAt = DateTime.UtcNow.ToString("o"),
|
||||
Files = new[]
|
||||
{
|
||||
new BundleFile { Path = "model.gguf", Digest = modelDigest, Size = modelContent.Length, Type = "weights" },
|
||||
new BundleFile { Path = "tokenizer.json", Digest = tokenizerDigest, Size = tokenizerContent.Length, Type = "tokenizer" }
|
||||
}
|
||||
};
|
||||
|
||||
var manifestJson = JsonSerializer.Serialize(manifest, new JsonSerializerOptions { WriteIndented = true });
|
||||
File.WriteAllText(Path.Combine(bundlePath, "manifest.json"), manifestJson);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Stub Implementations
|
||||
|
||||
private sealed class StubLlmProvider : ILlmProvider
|
||||
{
|
||||
public string ProviderId => ProviderIdOverride ?? "stub";
|
||||
public string? ProviderIdOverride { get; set; }
|
||||
public bool IsAvailableResult { get; set; } = true;
|
||||
|
||||
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(IsAvailableResult);
|
||||
|
||||
public Task<LlmCompletionResult> CompleteAsync(
|
||||
LlmCompletionRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Generate deterministic output based on input hash
|
||||
using var sha = SHA256.Create();
|
||||
var inputHash = Convert.ToHexStringLower(
|
||||
sha.ComputeHash(Encoding.UTF8.GetBytes(
|
||||
$"{request.SystemPrompt}||{request.UserPrompt}||{request.Seed}")));
|
||||
|
||||
var content = $"Deterministic response for input hash: {inputHash[..16]}";
|
||||
|
||||
return Task.FromResult(new LlmCompletionResult
|
||||
{
|
||||
Content = content,
|
||||
ModelId = "stub-model",
|
||||
ProviderId = ProviderId,
|
||||
Deterministic = request.Temperature == 0,
|
||||
InputTokens = request.UserPrompt.Length / 4,
|
||||
OutputTokens = content.Length / 4,
|
||||
FinishReason = "stop",
|
||||
RequestId = request.RequestId
|
||||
});
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<LlmStreamChunk> CompleteStreamAsync(
|
||||
LlmCompletionRequest request,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
var words = new[] { "This ", "is ", "a ", "streaming ", "response." };
|
||||
|
||||
foreach (var word in words)
|
||||
{
|
||||
await Task.Delay(10, cancellationToken);
|
||||
yield return new LlmStreamChunk { Content = word, IsFinal = false };
|
||||
}
|
||||
|
||||
yield return new LlmStreamChunk { Content = "", IsFinal = true, FinishReason = "stop" };
|
||||
}
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
private sealed class CallCountingLlmProvider : ILlmProvider
|
||||
{
|
||||
public string ProviderId => "counting";
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult(true);
|
||||
|
||||
public Task<LlmCompletionResult> CompleteAsync(
|
||||
LlmCompletionRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
CallCount++;
|
||||
return Task.FromResult(new LlmCompletionResult
|
||||
{
|
||||
Content = $"Response #{CallCount}",
|
||||
ModelId = "counting-model",
|
||||
ProviderId = ProviderId,
|
||||
Deterministic = request.Temperature == 0,
|
||||
OutputTokens = 5
|
||||
});
|
||||
}
|
||||
|
||||
public async IAsyncEnumerable<LlmStreamChunk> CompleteStreamAsync(
|
||||
LlmCompletionRequest request,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken = default)
|
||||
{
|
||||
CallCount++;
|
||||
yield return new LlmStreamChunk { Content = "Response", IsFinal = true };
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
|
||||
private sealed class StubLlmProviderFactory : ILlmProviderFactory
|
||||
{
|
||||
private readonly Dictionary<string, ILlmProvider> _providers;
|
||||
|
||||
public StubLlmProviderFactory(Dictionary<string, ILlmProvider> providers)
|
||||
{
|
||||
_providers = providers;
|
||||
}
|
||||
|
||||
public IReadOnlyList<string> AvailableProviders => _providers.Keys.ToList();
|
||||
|
||||
public ILlmProvider GetProvider(string providerId)
|
||||
{
|
||||
if (_providers.TryGetValue(providerId, out var provider))
|
||||
return provider;
|
||||
|
||||
throw new InvalidOperationException($"Provider '{providerId}' not found");
|
||||
}
|
||||
|
||||
public ILlmProvider GetDefaultProvider() => _providers.Values.First();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,834 @@
|
||||
using FluentAssertions;
|
||||
using StellaOps.AdvisoryAI.PolicyStudio;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for Policy Studio NL→rule→test round-trip and conflict detection.
|
||||
/// Sprint: SPRINT_20251226_017_AI_policy_copilot
|
||||
/// Task: POLICY-25
|
||||
/// </summary>
|
||||
public sealed class PolicyStudioIntegrationTests
|
||||
{
|
||||
#region NL → Intent → Rule Round-Trip Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ParseAndGenerate_OverrideRule_ProducesValidLatticeRule()
|
||||
{
|
||||
// Arrange
|
||||
var parser = new StubPolicyIntentParser();
|
||||
var generator = new StubPolicyRuleGenerator();
|
||||
var synthesizer = new StubTestCaseSynthesizer();
|
||||
|
||||
var naturalLanguage = "Block all critical vulnerabilities that are reachable";
|
||||
|
||||
// Act - Parse NL to intent
|
||||
var parseResult = await parser.ParseAsync(naturalLanguage);
|
||||
parseResult.Success.Should().BeTrue();
|
||||
parseResult.Intent.IntentType.Should().Be(PolicyIntentType.OverrideRule);
|
||||
|
||||
// Act - Generate rules from intent
|
||||
var ruleResult = await generator.GenerateAsync(parseResult.Intent);
|
||||
ruleResult.Success.Should().BeTrue();
|
||||
ruleResult.Rules.Should().NotBeEmpty();
|
||||
|
||||
// Assert - Rules have correct structure
|
||||
var rule = ruleResult.Rules[0];
|
||||
rule.LatticeExpression.Should().Contain("REACHABLE");
|
||||
rule.Disposition.Should().Be("block");
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ParseAndGenerate_ExceptionRule_ProducesValidLatticeRule()
|
||||
{
|
||||
// Arrange
|
||||
var parser = new StubPolicyIntentParser();
|
||||
var generator = new StubPolicyRuleGenerator();
|
||||
|
||||
var naturalLanguage = "Allow vulnerabilities with vendor VEX not_affected status";
|
||||
|
||||
// Act
|
||||
var parseResult = await parser.ParseAsync(naturalLanguage, new PolicyParseContext
|
||||
{
|
||||
DefaultScope = "all"
|
||||
});
|
||||
|
||||
var ruleResult = await generator.GenerateAsync(parseResult.Intent);
|
||||
|
||||
// Assert
|
||||
ruleResult.Success.Should().BeTrue();
|
||||
var rule = ruleResult.Rules[0];
|
||||
rule.Disposition.Should().Be("allow");
|
||||
rule.Conditions.Should().Contain(c => c.Field == "vex_status");
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task FullRoundTrip_NLToRuleToTest_ProducesValidTestCases()
|
||||
{
|
||||
// Arrange
|
||||
var parser = new StubPolicyIntentParser();
|
||||
var generator = new StubPolicyRuleGenerator();
|
||||
var synthesizer = new StubTestCaseSynthesizer();
|
||||
|
||||
var naturalLanguage = "Block critical reachable vulnerabilities without VEX";
|
||||
|
||||
// Act - Full round-trip
|
||||
var parseResult = await parser.ParseAsync(naturalLanguage);
|
||||
var ruleResult = await generator.GenerateAsync(parseResult.Intent);
|
||||
var testCases = await synthesizer.SynthesizeAsync(ruleResult.Rules);
|
||||
|
||||
// Assert
|
||||
testCases.Should().NotBeEmpty();
|
||||
testCases.Should().Contain(t => t.Type == TestCaseType.Positive);
|
||||
testCases.Should().Contain(t => t.Type == TestCaseType.Negative);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("Block all high severity findings", PolicyIntentType.OverrideRule)]
|
||||
[InlineData("Escalate critical vulnerabilities to security team", PolicyIntentType.EscalationRule)]
|
||||
[InlineData("Allow exceptions for internal-only services", PolicyIntentType.ExceptionCondition)]
|
||||
[InlineData("Set severity threshold to 7.0 for blocking", PolicyIntentType.ThresholdRule)]
|
||||
public async Task ParseAsync_RecognizesIntentTypes(string input, PolicyIntentType expectedType)
|
||||
{
|
||||
// Arrange
|
||||
var parser = new StubPolicyIntentParser(expectedType);
|
||||
|
||||
// Act
|
||||
var result = await parser.ParseAsync(input);
|
||||
|
||||
// Assert
|
||||
result.Success.Should().BeTrue();
|
||||
result.Intent.IntentType.Should().Be(expectedType);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Conflict Detection Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ValidateAsync_DetectsConflictingRules()
|
||||
{
|
||||
// Arrange
|
||||
var generator = new StubPolicyRuleGenerator();
|
||||
|
||||
var conflictingRules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "REACHABLE ∧ PRESENT", "block", priority: 10),
|
||||
CreateRule("rule-2", "REACHABLE ∧ PRESENT", "allow", priority: 20)
|
||||
};
|
||||
|
||||
// Act
|
||||
var validationResult = await generator.ValidateAsync(conflictingRules);
|
||||
|
||||
// Assert
|
||||
validationResult.Valid.Should().BeFalse();
|
||||
validationResult.Conflicts.Should().NotBeEmpty();
|
||||
validationResult.Conflicts[0].RuleId1.Should().Be("rule-1");
|
||||
validationResult.Conflicts[0].RuleId2.Should().Be("rule-2");
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ValidateAsync_NoConflict_WhenDifferentConditions()
|
||||
{
|
||||
// Arrange
|
||||
var generator = new StubPolicyRuleGenerator();
|
||||
|
||||
var nonConflictingRules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "REACHABLE ∧ PRESENT", "block", priority: 10),
|
||||
CreateRule("rule-2", "¬REACHABLE ∧ PRESENT", "allow", priority: 20)
|
||||
};
|
||||
|
||||
// Act
|
||||
var validationResult = await generator.ValidateAsync(nonConflictingRules);
|
||||
|
||||
// Assert
|
||||
validationResult.Conflicts.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ValidateAsync_DetectsUnreachableConditions()
|
||||
{
|
||||
// Arrange
|
||||
var generator = new StubPolicyRuleGenerator();
|
||||
|
||||
var rules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "REACHABLE ∧ ¬REACHABLE", "block", priority: 10) // Contradiction
|
||||
};
|
||||
|
||||
// Act
|
||||
var validationResult = await generator.ValidateAsync(rules);
|
||||
|
||||
// Assert
|
||||
validationResult.UnreachableConditions.Should().NotBeEmpty();
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ValidateAsync_ReportsCoverageMetric()
|
||||
{
|
||||
// Arrange
|
||||
var generator = new StubPolicyRuleGenerator();
|
||||
|
||||
var rules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "REACHABLE", "block", priority: 10),
|
||||
CreateRule("rule-2", "PRESENT", "warn", priority: 20),
|
||||
CreateRule("rule-3", "FIXED", "allow", priority: 30)
|
||||
};
|
||||
|
||||
// Act
|
||||
var validationResult = await generator.ValidateAsync(rules);
|
||||
|
||||
// Assert
|
||||
validationResult.Coverage.Should().BeGreaterThan(0);
|
||||
validationResult.Coverage.Should().BeLessThanOrEqualTo(1.0);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Test Case Synthesis Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SynthesizeAsync_GeneratesPositiveTests()
|
||||
{
|
||||
// Arrange
|
||||
var synthesizer = new StubTestCaseSynthesizer();
|
||||
var rules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "REACHABLE ∧ PRESENT", "block", priority: 10)
|
||||
};
|
||||
|
||||
// Act
|
||||
var testCases = await synthesizer.SynthesizeAsync(rules);
|
||||
|
||||
// Assert
|
||||
var positiveTests = testCases.Where(t => t.Type == TestCaseType.Positive).ToList();
|
||||
positiveTests.Should().NotBeEmpty();
|
||||
positiveTests.Should().AllSatisfy(t =>
|
||||
{
|
||||
t.ExpectedDisposition.Should().Be("block");
|
||||
t.TargetRuleIds.Should().Contain("rule-1");
|
||||
});
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SynthesizeAsync_GeneratesNegativeTests()
|
||||
{
|
||||
// Arrange
|
||||
var synthesizer = new StubTestCaseSynthesizer();
|
||||
var rules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "REACHABLE ∧ PRESENT", "block", priority: 10)
|
||||
};
|
||||
|
||||
// Act
|
||||
var testCases = await synthesizer.SynthesizeAsync(rules);
|
||||
|
||||
// Assert
|
||||
var negativeTests = testCases.Where(t => t.Type == TestCaseType.Negative).ToList();
|
||||
negativeTests.Should().NotBeEmpty();
|
||||
negativeTests.Should().AllSatisfy(t =>
|
||||
{
|
||||
t.Description.Should().NotBeNullOrEmpty();
|
||||
});
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SynthesizeAsync_GeneratesBoundaryTests()
|
||||
{
|
||||
// Arrange
|
||||
var synthesizer = new StubTestCaseSynthesizer();
|
||||
var rules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "cvss_score >= 7.0", "block", priority: 10)
|
||||
};
|
||||
|
||||
// Act
|
||||
var testCases = await synthesizer.SynthesizeAsync(rules);
|
||||
|
||||
// Assert
|
||||
var boundaryTests = testCases.Where(t => t.Type == TestCaseType.Boundary).ToList();
|
||||
boundaryTests.Should().NotBeEmpty();
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task SynthesizeAsync_GeneratesConflictTests_ForOverlappingRules()
|
||||
{
|
||||
// Arrange
|
||||
var synthesizer = new StubTestCaseSynthesizer();
|
||||
var rules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "REACHABLE", "block", priority: 10),
|
||||
CreateRule("rule-2", "REACHABLE ∧ HAS_VEX", "allow", priority: 20)
|
||||
};
|
||||
|
||||
// Act
|
||||
var testCases = await synthesizer.SynthesizeAsync(rules);
|
||||
|
||||
// Assert
|
||||
var conflictTests = testCases.Where(t => t.Type == TestCaseType.Conflict).ToList();
|
||||
conflictTests.Should().NotBeEmpty();
|
||||
conflictTests.Should().AllSatisfy(t =>
|
||||
{
|
||||
t.TargetRuleIds.Count.Should().BeGreaterThan(1);
|
||||
});
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RunTestsAsync_PassesWithMatchingRules()
|
||||
{
|
||||
// Arrange
|
||||
var synthesizer = new StubTestCaseSynthesizer();
|
||||
var rules = new List<LatticeRule>
|
||||
{
|
||||
CreateRule("rule-1", "REACHABLE", "block", priority: 10)
|
||||
};
|
||||
|
||||
var testCases = await synthesizer.SynthesizeAsync(rules);
|
||||
|
||||
// Act
|
||||
var result = await synthesizer.RunTestsAsync(testCases, rules);
|
||||
|
||||
// Assert
|
||||
result.Success.Should().BeTrue();
|
||||
result.Passed.Should().Be(result.Total);
|
||||
result.Failed.Should().Be(0);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Edge Cases
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ParseAsync_WithAmbiguousInput_ReturnsAlternatives()
|
||||
{
|
||||
// Arrange
|
||||
var parser = new StubPolicyIntentParser(ambiguous: true);
|
||||
|
||||
var ambiguousInput = "Block vulnerabilities in production";
|
||||
|
||||
// Act
|
||||
var result = await parser.ParseAsync(ambiguousInput);
|
||||
|
||||
// Assert
|
||||
result.Intent.Alternatives.Should().NotBeNullOrEmpty();
|
||||
result.Intent.ClarifyingQuestions.Should().NotBeNullOrEmpty();
|
||||
result.Intent.Confidence.Should().BeLessThan(0.9);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GenerateAsync_WithEmptyConditions_ReturnsError()
|
||||
{
|
||||
// Arrange
|
||||
var generator = new StubPolicyRuleGenerator();
|
||||
var intent = new PolicyIntent
|
||||
{
|
||||
IntentId = "empty-intent",
|
||||
IntentType = PolicyIntentType.OverrideRule,
|
||||
OriginalInput = "Block everything",
|
||||
Conditions = [],
|
||||
Actions = [],
|
||||
Scope = "all",
|
||||
Priority = 100,
|
||||
Confidence = 0.5
|
||||
};
|
||||
|
||||
// Act
|
||||
var result = await generator.GenerateAsync(intent);
|
||||
|
||||
// Assert
|
||||
result.Success.Should().BeFalse();
|
||||
result.Errors.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ClarifyAsync_UpdatesIntentWithClarification()
|
||||
{
|
||||
// Arrange
|
||||
var parser = new StubPolicyIntentParser(ambiguous: true);
|
||||
var initialResult = await parser.ParseAsync("Block vulnerabilities");
|
||||
|
||||
// Act
|
||||
var clarifiedResult = await parser.ClarifyAsync(
|
||||
initialResult.Intent.IntentId,
|
||||
"Only critical severity");
|
||||
|
||||
// Assert
|
||||
clarifiedResult.Intent.Confidence.Should().BeGreaterThan(initialResult.Intent.Confidence);
|
||||
clarifiedResult.Intent.ClarifyingQuestions.Should().BeNullOrEmpty();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static LatticeRule CreateRule(string ruleId, string expression, string disposition, int priority)
|
||||
=> new()
|
||||
{
|
||||
RuleId = ruleId,
|
||||
Name = $"Test Rule {ruleId}",
|
||||
Description = $"Test rule with expression: {expression}",
|
||||
LatticeExpression = expression,
|
||||
Conditions = ParseConditions(expression),
|
||||
Disposition = disposition,
|
||||
Priority = priority,
|
||||
Scope = "all",
|
||||
Enabled = true
|
||||
};
|
||||
|
||||
private static IReadOnlyList<PolicyCondition> ParseConditions(string expression)
|
||||
{
|
||||
var conditions = new List<PolicyCondition>();
|
||||
|
||||
if (expression.Contains("REACHABLE", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
conditions.Add(new PolicyCondition
|
||||
{
|
||||
Field = "reachable",
|
||||
Operator = "equals",
|
||||
Value = true,
|
||||
Connector = expression.Contains("∧") ? "and" : null
|
||||
});
|
||||
}
|
||||
|
||||
if (expression.Contains("PRESENT", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
conditions.Add(new PolicyCondition
|
||||
{
|
||||
Field = "present",
|
||||
Operator = "equals",
|
||||
Value = true
|
||||
});
|
||||
}
|
||||
|
||||
if (expression.Contains("cvss_score", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
conditions.Add(new PolicyCondition
|
||||
{
|
||||
Field = "cvss_score",
|
||||
Operator = "greater_than_or_equal",
|
||||
Value = 7.0
|
||||
});
|
||||
}
|
||||
|
||||
return conditions;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Stub Implementations
|
||||
|
||||
private sealed class StubPolicyIntentParser : IPolicyIntentParser
|
||||
{
|
||||
private readonly PolicyIntentType _defaultType;
|
||||
private readonly bool _ambiguous;
|
||||
private readonly Dictionary<string, PolicyIntent> _intents = new();
|
||||
|
||||
public StubPolicyIntentParser(
|
||||
PolicyIntentType defaultType = PolicyIntentType.OverrideRule,
|
||||
bool ambiguous = false)
|
||||
{
|
||||
_defaultType = defaultType;
|
||||
_ambiguous = ambiguous;
|
||||
}
|
||||
|
||||
public Task<PolicyParseResult> ParseAsync(
|
||||
string naturalLanguageInput,
|
||||
PolicyParseContext? context = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var intentId = $"intent-{Guid.NewGuid():N}";
|
||||
var confidence = _ambiguous ? 0.7 : 0.95;
|
||||
|
||||
var conditions = new List<PolicyCondition>();
|
||||
|
||||
if (naturalLanguageInput.Contains("critical", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
conditions.Add(new PolicyCondition
|
||||
{
|
||||
Field = "severity",
|
||||
Operator = "equals",
|
||||
Value = "critical"
|
||||
});
|
||||
}
|
||||
|
||||
if (naturalLanguageInput.Contains("reachable", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
conditions.Add(new PolicyCondition
|
||||
{
|
||||
Field = "reachable",
|
||||
Operator = "equals",
|
||||
Value = true
|
||||
});
|
||||
}
|
||||
|
||||
if (naturalLanguageInput.Contains("VEX", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
conditions.Add(new PolicyCondition
|
||||
{
|
||||
Field = "vex_status",
|
||||
Operator = "equals",
|
||||
Value = "not_affected"
|
||||
});
|
||||
}
|
||||
|
||||
var intent = new PolicyIntent
|
||||
{
|
||||
IntentId = intentId,
|
||||
IntentType = _defaultType,
|
||||
OriginalInput = naturalLanguageInput,
|
||||
Conditions = conditions,
|
||||
Actions = [new PolicyAction
|
||||
{
|
||||
ActionType = "set_verdict",
|
||||
Parameters = new Dictionary<string, object> { ["verdict"] = "block" }
|
||||
}],
|
||||
Scope = context?.DefaultScope ?? "all",
|
||||
Priority = 100,
|
||||
Confidence = confidence,
|
||||
Alternatives = _ambiguous ? [CreateAlternativeIntent(naturalLanguageInput)] : null,
|
||||
ClarifyingQuestions = _ambiguous ? ["What severity levels should be affected?"] : null
|
||||
};
|
||||
|
||||
_intents[intentId] = intent;
|
||||
|
||||
return Task.FromResult(new PolicyParseResult
|
||||
{
|
||||
Intent = intent,
|
||||
Success = true,
|
||||
ModelId = "stub-parser-v1",
|
||||
ParsedAt = DateTime.UtcNow.ToString("O")
|
||||
});
|
||||
}
|
||||
|
||||
public Task<PolicyParseResult> ClarifyAsync(
|
||||
string intentId,
|
||||
string clarification,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var original = _intents.GetValueOrDefault(intentId);
|
||||
if (original is null)
|
||||
{
|
||||
throw new InvalidOperationException($"Intent {intentId} not found");
|
||||
}
|
||||
|
||||
var clarified = original with
|
||||
{
|
||||
Confidence = 0.95,
|
||||
ClarifyingQuestions = null,
|
||||
Alternatives = null
|
||||
};
|
||||
|
||||
return Task.FromResult(new PolicyParseResult
|
||||
{
|
||||
Intent = clarified,
|
||||
Success = true,
|
||||
ModelId = "stub-parser-v1",
|
||||
ParsedAt = DateTime.UtcNow.ToString("O")
|
||||
});
|
||||
}
|
||||
|
||||
private PolicyIntent CreateAlternativeIntent(string input) => new()
|
||||
{
|
||||
IntentId = $"alt-{Guid.NewGuid():N}",
|
||||
IntentType = PolicyIntentType.ExceptionCondition,
|
||||
OriginalInput = input,
|
||||
Conditions = [],
|
||||
Actions = [],
|
||||
Scope = "all",
|
||||
Priority = 50,
|
||||
Confidence = 0.5
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class StubPolicyRuleGenerator : IPolicyRuleGenerator
|
||||
{
|
||||
public Task<RuleGenerationResult> GenerateAsync(
|
||||
PolicyIntent intent,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (intent.Conditions.Count == 0)
|
||||
{
|
||||
return Task.FromResult(new RuleGenerationResult
|
||||
{
|
||||
Rules = [],
|
||||
Success = false,
|
||||
Warnings = [],
|
||||
Errors = ["Intent must have at least one condition"],
|
||||
IntentId = intent.IntentId,
|
||||
GeneratedAt = DateTime.UtcNow.ToString("O")
|
||||
});
|
||||
}
|
||||
|
||||
var expression = BuildLatticeExpression(intent.Conditions);
|
||||
var disposition = intent.Actions.FirstOrDefault()?.Parameters.GetValueOrDefault("verdict")?.ToString() ?? "warn";
|
||||
|
||||
var rule = new LatticeRule
|
||||
{
|
||||
RuleId = $"rule-{Guid.NewGuid():N}",
|
||||
Name = $"Generated from: {intent.OriginalInput[..Math.Min(30, intent.OriginalInput.Length)]}",
|
||||
Description = intent.OriginalInput,
|
||||
LatticeExpression = expression,
|
||||
Conditions = intent.Conditions,
|
||||
Disposition = disposition,
|
||||
Priority = intent.Priority,
|
||||
Scope = intent.Scope,
|
||||
Enabled = true
|
||||
};
|
||||
|
||||
return Task.FromResult(new RuleGenerationResult
|
||||
{
|
||||
Rules = [rule],
|
||||
Success = true,
|
||||
Warnings = [],
|
||||
IntentId = intent.IntentId,
|
||||
GeneratedAt = DateTime.UtcNow.ToString("O")
|
||||
});
|
||||
}
|
||||
|
||||
public Task<RuleValidationResult> ValidateAsync(
|
||||
IReadOnlyList<LatticeRule> rules,
|
||||
IReadOnlyList<string>? existingRuleIds = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var conflicts = new List<RuleConflict>();
|
||||
var unreachable = new List<string>();
|
||||
|
||||
// Check for conflicts
|
||||
for (int i = 0; i < rules.Count; i++)
|
||||
{
|
||||
for (int j = i + 1; j < rules.Count; j++)
|
||||
{
|
||||
if (HasConflict(rules[i], rules[j]))
|
||||
{
|
||||
conflicts.Add(new RuleConflict
|
||||
{
|
||||
RuleId1 = rules[i].RuleId,
|
||||
RuleId2 = rules[j].RuleId,
|
||||
Description = "Rules have overlapping conditions with different dispositions",
|
||||
SuggestedResolution = "Adjust priority or narrow conditions",
|
||||
Severity = "error"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check for unreachable conditions (contradictions)
|
||||
if (rules[i].LatticeExpression.Contains("∧ ¬") &&
|
||||
rules[i].LatticeExpression.Split("∧").Any(p =>
|
||||
p.Trim().StartsWith("¬") && rules[i].LatticeExpression.Contains(p.Trim()[1..])))
|
||||
{
|
||||
unreachable.Add($"Rule {rules[i].RuleId} has contradictory conditions");
|
||||
}
|
||||
}
|
||||
|
||||
var coverage = Math.Min(1.0, rules.Count * 0.2);
|
||||
|
||||
return Task.FromResult(new RuleValidationResult
|
||||
{
|
||||
Valid = conflicts.Count == 0 && unreachable.Count == 0,
|
||||
Conflicts = conflicts,
|
||||
UnreachableConditions = unreachable,
|
||||
PotentialLoops = [],
|
||||
Coverage = coverage
|
||||
});
|
||||
}
|
||||
|
||||
private static bool HasConflict(LatticeRule rule1, LatticeRule rule2)
|
||||
{
|
||||
// Simplified conflict detection
|
||||
var sameConditions = rule1.LatticeExpression == rule2.LatticeExpression;
|
||||
var differentDispositions = rule1.Disposition != rule2.Disposition;
|
||||
return sameConditions && differentDispositions;
|
||||
}
|
||||
|
||||
private static string BuildLatticeExpression(IReadOnlyList<PolicyCondition> conditions)
|
||||
{
|
||||
var parts = conditions.Select(c =>
|
||||
{
|
||||
var atom = c.Field.ToUpperInvariant() switch
|
||||
{
|
||||
"REACHABLE" => "REACHABLE",
|
||||
"PRESENT" => "PRESENT",
|
||||
"SEVERITY" => c.Value?.ToString()?.ToUpperInvariant() ?? "CRITICAL",
|
||||
"VEX_STATUS" => "HAS_VEX",
|
||||
_ => c.Field.ToUpperInvariant()
|
||||
};
|
||||
|
||||
return c.Operator == "not_equals" || c.Value?.Equals(false) == true
|
||||
? $"¬{atom}"
|
||||
: atom;
|
||||
});
|
||||
|
||||
return string.Join(" ∧ ", parts);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubTestCaseSynthesizer : ITestCaseSynthesizer
|
||||
{
|
||||
public Task<IReadOnlyList<PolicyTestCase>> SynthesizeAsync(
|
||||
IReadOnlyList<LatticeRule> rules,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var testCases = new List<PolicyTestCase>();
|
||||
var testId = 0;
|
||||
|
||||
foreach (var rule in rules)
|
||||
{
|
||||
// Positive test
|
||||
testCases.Add(new PolicyTestCase
|
||||
{
|
||||
TestCaseId = $"test-{++testId}",
|
||||
Name = $"Positive test for {rule.Name}",
|
||||
Type = TestCaseType.Positive,
|
||||
Input = BuildPositiveInput(rule),
|
||||
ExpectedDisposition = rule.Disposition,
|
||||
TargetRuleIds = [rule.RuleId],
|
||||
Description = $"Verifies rule matches when conditions are met"
|
||||
});
|
||||
|
||||
// Negative test
|
||||
testCases.Add(new PolicyTestCase
|
||||
{
|
||||
TestCaseId = $"test-{++testId}",
|
||||
Name = $"Negative test for {rule.Name}",
|
||||
Type = TestCaseType.Negative,
|
||||
Input = BuildNegativeInput(rule),
|
||||
ExpectedDisposition = "no_match",
|
||||
TargetRuleIds = [rule.RuleId],
|
||||
Description = $"Verifies rule does not match when conditions are not met"
|
||||
});
|
||||
|
||||
// Boundary test for numeric conditions
|
||||
if (rule.LatticeExpression.Contains("cvss_score") || rule.LatticeExpression.Contains(">="))
|
||||
{
|
||||
testCases.Add(new PolicyTestCase
|
||||
{
|
||||
TestCaseId = $"test-{++testId}",
|
||||
Name = $"Boundary test for {rule.Name}",
|
||||
Type = TestCaseType.Boundary,
|
||||
Input = BuildBoundaryInput(rule),
|
||||
ExpectedDisposition = rule.Disposition,
|
||||
TargetRuleIds = [rule.RuleId],
|
||||
Description = $"Verifies rule at boundary values"
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Conflict tests for overlapping rules
|
||||
for (int i = 0; i < rules.Count; i++)
|
||||
{
|
||||
for (int j = i + 1; j < rules.Count; j++)
|
||||
{
|
||||
if (RulesOverlap(rules[i], rules[j]))
|
||||
{
|
||||
testCases.Add(new PolicyTestCase
|
||||
{
|
||||
TestCaseId = $"test-{++testId}",
|
||||
Name = $"Conflict test: {rules[i].Name} vs {rules[j].Name}",
|
||||
Type = TestCaseType.Conflict,
|
||||
Input = BuildOverlapInput(rules[i], rules[j]),
|
||||
ExpectedDisposition = rules[i].Priority > rules[j].Priority
|
||||
? rules[i].Disposition
|
||||
: rules[j].Disposition,
|
||||
TargetRuleIds = [rules[i].RuleId, rules[j].RuleId],
|
||||
Description = $"Verifies priority resolution when both rules match"
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult<IReadOnlyList<PolicyTestCase>>(testCases);
|
||||
}
|
||||
|
||||
public Task<TestRunResult> RunTestsAsync(
|
||||
IReadOnlyList<PolicyTestCase> testCases,
|
||||
IReadOnlyList<LatticeRule> rules,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var results = new List<TestCaseResult>();
|
||||
|
||||
foreach (var testCase in testCases)
|
||||
{
|
||||
var passed = true; // Simplified - stub always passes
|
||||
results.Add(new TestCaseResult
|
||||
{
|
||||
TestCaseId = testCase.TestCaseId,
|
||||
Passed = passed,
|
||||
Expected = testCase.ExpectedDisposition,
|
||||
Actual = testCase.ExpectedDisposition
|
||||
});
|
||||
}
|
||||
|
||||
return Task.FromResult(new TestRunResult
|
||||
{
|
||||
Total = testCases.Count,
|
||||
Passed = results.Count(r => r.Passed),
|
||||
Failed = results.Count(r => !r.Passed),
|
||||
Results = results,
|
||||
RunAt = DateTime.UtcNow.ToString("O")
|
||||
});
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, object> BuildPositiveInput(LatticeRule rule)
|
||||
{
|
||||
var input = new Dictionary<string, object>();
|
||||
if (rule.LatticeExpression.Contains("REACHABLE")) input["reachable"] = true;
|
||||
if (rule.LatticeExpression.Contains("PRESENT")) input["present"] = true;
|
||||
if (rule.LatticeExpression.Contains("HAS_VEX")) input["has_vex"] = true;
|
||||
return input;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, object> BuildNegativeInput(LatticeRule rule)
|
||||
{
|
||||
var input = new Dictionary<string, object>();
|
||||
if (rule.LatticeExpression.Contains("REACHABLE")) input["reachable"] = false;
|
||||
if (rule.LatticeExpression.Contains("PRESENT")) input["present"] = false;
|
||||
return input;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, object> BuildBoundaryInput(LatticeRule rule)
|
||||
{
|
||||
return new Dictionary<string, object>
|
||||
{
|
||||
["cvss_score"] = 7.0
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, object> BuildOverlapInput(LatticeRule rule1, LatticeRule rule2)
|
||||
{
|
||||
var input = new Dictionary<string, object>();
|
||||
input["reachable"] = true;
|
||||
input["present"] = true;
|
||||
input["has_vex"] = true;
|
||||
return input;
|
||||
}
|
||||
|
||||
private static bool RulesOverlap(LatticeRule rule1, LatticeRule rule2)
|
||||
{
|
||||
// Simplified overlap detection
|
||||
return rule1.LatticeExpression.Contains("REACHABLE") &&
|
||||
rule2.LatticeExpression.Contains("REACHABLE");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,791 @@
|
||||
using StellaOps.AdvisoryAI.Remediation;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for remediation plan generation and PR creation.
|
||||
/// Sprint: SPRINT_20251226_016_AI_remedy_autopilot
|
||||
/// Task: REMEDY-25
|
||||
/// </summary>
|
||||
public sealed class RemediationIntegrationTests
|
||||
{
|
||||
#region Plan Generation Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_ValidRequest_ReturnsPlan()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(plan);
|
||||
Assert.Equal(request.FindingId, plan.Request.FindingId);
|
||||
Assert.NotEmpty(plan.Steps);
|
||||
Assert.NotEmpty(plan.PlanId);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_BumpRemediation_GeneratesBumpSteps()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest() with
|
||||
{
|
||||
RemediationType = RemediationType.Bump,
|
||||
ComponentPurl = "pkg:npm/lodash@4.17.20"
|
||||
};
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Contains(plan.Steps, s => s.ActionType == "update_package");
|
||||
Assert.True(plan.ExpectedDelta.Upgraded.Count > 0);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_UpgradeRemediation_GeneratesUpgradeSteps()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest() with
|
||||
{
|
||||
RemediationType = RemediationType.Upgrade,
|
||||
ComponentPurl = "pkg:oci/alpine@3.18"
|
||||
};
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Contains(plan.Steps, s => s.ActionType == "update_base_image");
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_ConfigRemediation_GeneratesConfigSteps()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest() with
|
||||
{
|
||||
RemediationType = RemediationType.Config,
|
||||
VulnerabilityId = "CVE-2021-44228" // Log4Shell
|
||||
};
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Contains(plan.Steps, s => s.ActionType == "update_config");
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_AssessesRiskCorrectly_PatchVersion()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner(patchVersionBump: true);
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(RemediationRisk.Low, plan.RiskAssessment);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_AssessesRiskCorrectly_MajorVersion()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner(majorVersionBump: true);
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(RemediationRisk.High, plan.RiskAssessment);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_IncludesExpectedSbomDelta()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(plan.ExpectedDelta);
|
||||
Assert.True(plan.ExpectedDelta.NetVulnerabilityChange < 0); // Should improve
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_IncludesTestRequirements()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(plan.TestRequirements);
|
||||
Assert.NotEmpty(plan.TestRequirements.TestSuites);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_IncludesInputHashes()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.NotEmpty(plan.InputHashes);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ValidatePlanAsync_ExistingPlan_ReturnsTrue()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest();
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Act
|
||||
var isValid = await planner.ValidatePlanAsync(plan.PlanId);
|
||||
|
||||
// Assert
|
||||
Assert.True(isValid);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ValidatePlanAsync_NonexistentPlan_ReturnsFalse()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
|
||||
// Act
|
||||
var isValid = await planner.ValidatePlanAsync("nonexistent-plan");
|
||||
|
||||
// Assert
|
||||
Assert.False(isValid);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GetPlanAsync_ExistingPlan_ReturnsPlan()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest();
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Act
|
||||
var retrieved = await planner.GetPlanAsync(plan.PlanId);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(retrieved);
|
||||
Assert.Equal(plan.PlanId, retrieved.PlanId);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region PR Generation Tests (Mocked SCM)
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePullRequestAsync_ValidPlan_CreatesPR()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator();
|
||||
var plan = CreateTestPlan();
|
||||
|
||||
// Act
|
||||
var result = await prGenerator.CreatePullRequestAsync(plan);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(result);
|
||||
Assert.NotEmpty(result.PrId);
|
||||
Assert.True(result.PrNumber > 0);
|
||||
Assert.NotEmpty(result.Url);
|
||||
Assert.NotEmpty(result.BranchName);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePullRequestAsync_SetsBranchNameFromPlan()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator();
|
||||
var plan = CreateTestPlan();
|
||||
|
||||
// Act
|
||||
var result = await prGenerator.CreatePullRequestAsync(plan);
|
||||
|
||||
// Assert
|
||||
Assert.Contains("stellaops-fix", result.BranchName);
|
||||
Assert.Contains(plan.Request.VulnerabilityId.ToLowerInvariant(), result.BranchName);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task CreatePullRequestAsync_InitialStatus_IsOpen()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator();
|
||||
var plan = CreateTestPlan();
|
||||
|
||||
// Act
|
||||
var result = await prGenerator.CreatePullRequestAsync(plan);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(PullRequestStatus.Open, result.Status);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GetStatusAsync_ExistingPR_ReturnsStatus()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator();
|
||||
var plan = CreateTestPlan();
|
||||
var pr = await prGenerator.CreatePullRequestAsync(plan);
|
||||
|
||||
// Act
|
||||
var status = await prGenerator.GetStatusAsync(pr.PrId);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(status);
|
||||
Assert.Equal(pr.PrId, status.PrId);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task UpdateWithDeltaVerdictAsync_UpdatesPR()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator();
|
||||
var plan = CreateTestPlan();
|
||||
var pr = await prGenerator.CreatePullRequestAsync(plan);
|
||||
|
||||
var deltaVerdict = new DeltaVerdictResult
|
||||
{
|
||||
Improved = true,
|
||||
VulnerabilitiesFixed = 3,
|
||||
VulnerabilitiesIntroduced = 0,
|
||||
VerdictId = "delta-001",
|
||||
ComputedAt = DateTime.UtcNow.ToString("o")
|
||||
};
|
||||
|
||||
// Act
|
||||
await prGenerator.UpdateWithDeltaVerdictAsync(pr.PrId, deltaVerdict);
|
||||
var updated = await prGenerator.GetStatusAsync(pr.PrId);
|
||||
|
||||
// Assert
|
||||
Assert.NotNull(updated.DeltaVerdict);
|
||||
Assert.Equal(3, updated.DeltaVerdict.VulnerabilitiesFixed);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task ClosePullRequestAsync_ClosesPR()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator();
|
||||
var plan = CreateTestPlan();
|
||||
var pr = await prGenerator.CreatePullRequestAsync(plan);
|
||||
|
||||
// Act
|
||||
await prGenerator.ClosePullRequestAsync(pr.PrId, "Superseded by manual fix");
|
||||
var status = await prGenerator.GetStatusAsync(pr.PrId);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(PullRequestStatus.Closed, status.Status);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void ScmType_GitHub_ReturnsCorrectType()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator { ScmTypeOverride = "github" };
|
||||
|
||||
// Assert
|
||||
Assert.Equal("github", prGenerator.ScmType);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void ScmType_GitLab_ReturnsCorrectType()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator { ScmTypeOverride = "gitlab" };
|
||||
|
||||
// Assert
|
||||
Assert.Equal("gitlab", prGenerator.ScmType);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void ScmType_AzureDevOps_ReturnsCorrectType()
|
||||
{
|
||||
// Arrange
|
||||
var prGenerator = new StubPullRequestGenerator { ScmTypeOverride = "azure-devops" };
|
||||
|
||||
// Assert
|
||||
Assert.Equal("azure-devops", prGenerator.ScmType);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Fallback Handling Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_BuildFails_SetsSuggestionAuthority()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner(buildWillFail: true);
|
||||
var request = CreateTestRequest() with { AutoCreatePr = true };
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(RemediationAuthority.Suggestion, plan.Authority);
|
||||
Assert.False(plan.PrReady);
|
||||
Assert.NotNull(plan.NotReadyReason);
|
||||
Assert.Contains("build", plan.NotReadyReason.ToLower());
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_TestsFail_SetsSuggestionAuthority()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner(testsWillFail: true);
|
||||
var request = CreateTestRequest() with { AutoCreatePr = true };
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(RemediationAuthority.Suggestion, plan.Authority);
|
||||
Assert.False(plan.PrReady);
|
||||
Assert.NotNull(plan.NotReadyReason);
|
||||
Assert.Contains("test", plan.NotReadyReason.ToLower());
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_NoAutoCreatePr_SetsDraftAuthority()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner();
|
||||
var request = CreateTestRequest() with { AutoCreatePr = false };
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(RemediationAuthority.Draft, plan.Authority);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_AllVerificationsPassed_SetsVerifiedAuthority()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner(allVerificationsPassed: true);
|
||||
var request = CreateTestRequest() with { AutoCreatePr = true };
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.Equal(RemediationAuthority.Verified, plan.Authority);
|
||||
Assert.True(plan.PrReady);
|
||||
Assert.Null(plan.NotReadyReason);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_BreakingChanges_ReducesConfidence()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner(hasBreakingChanges: true);
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.True(plan.ConfidenceScore < 0.8);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Confidence Score Tests
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_PatchVersion_HighConfidence()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner(patchVersionBump: true);
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.True(plan.ConfidenceScore >= 0.9);
|
||||
}
|
||||
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GeneratePlanAsync_MajorVersion_LowerConfidence()
|
||||
{
|
||||
// Arrange
|
||||
var planner = new StubRemediationPlanner(majorVersionBump: true);
|
||||
var request = CreateTestRequest();
|
||||
|
||||
// Act
|
||||
var plan = await planner.GeneratePlanAsync(request);
|
||||
|
||||
// Assert
|
||||
Assert.True(plan.ConfidenceScore < 0.7);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Helper Methods
|
||||
|
||||
private static RemediationPlanRequest CreateTestRequest()
|
||||
{
|
||||
return new RemediationPlanRequest
|
||||
{
|
||||
FindingId = "finding-001",
|
||||
ArtifactDigest = "sha256:abc123",
|
||||
VulnerabilityId = "CVE-2024-1234",
|
||||
ComponentPurl = "pkg:npm/lodash@4.17.20",
|
||||
RemediationType = RemediationType.Auto,
|
||||
RepositoryUrl = "https://github.com/test/repo",
|
||||
TargetBranch = "main",
|
||||
AutoCreatePr = false,
|
||||
CorrelationId = Guid.NewGuid().ToString()
|
||||
};
|
||||
}
|
||||
|
||||
private static RemediationPlan CreateTestPlan()
|
||||
{
|
||||
return new RemediationPlan
|
||||
{
|
||||
PlanId = $"plan-{Guid.NewGuid():N}",
|
||||
Request = CreateTestRequest(),
|
||||
Steps = new[]
|
||||
{
|
||||
new RemediationStep
|
||||
{
|
||||
Order = 1,
|
||||
ActionType = "update_package",
|
||||
FilePath = "package.json",
|
||||
Description = "Update lodash from 4.17.20 to 4.17.21",
|
||||
PreviousValue = "4.17.20",
|
||||
NewValue = "4.17.21",
|
||||
Risk = RemediationRisk.Low
|
||||
}
|
||||
},
|
||||
ExpectedDelta = new ExpectedSbomDelta
|
||||
{
|
||||
Added = Array.Empty<string>(),
|
||||
Removed = Array.Empty<string>(),
|
||||
Upgraded = new Dictionary<string, string>
|
||||
{
|
||||
["pkg:npm/lodash@4.17.20"] = "pkg:npm/lodash@4.17.21"
|
||||
},
|
||||
NetVulnerabilityChange = -1
|
||||
},
|
||||
RiskAssessment = RemediationRisk.Low,
|
||||
TestRequirements = new RemediationTestRequirements
|
||||
{
|
||||
TestSuites = new[] { "unit", "integration" },
|
||||
MinCoverage = 80,
|
||||
RequireAllPass = true,
|
||||
Timeout = TimeSpan.FromMinutes(15)
|
||||
},
|
||||
Authority = RemediationAuthority.Draft,
|
||||
PrReady = false,
|
||||
ConfidenceScore = 0.92,
|
||||
ModelId = "test-model",
|
||||
GeneratedAt = DateTime.UtcNow.ToString("o"),
|
||||
InputHashes = new[] { "hash1", "hash2" },
|
||||
EvidenceRefs = new[] { "evidence/sbom-001", "evidence/vuln-001" }
|
||||
};
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Stub Implementations
|
||||
|
||||
private sealed class StubRemediationPlanner : IRemediationPlanner
|
||||
{
|
||||
private readonly Dictionary<string, RemediationPlan> _plans = new();
|
||||
private readonly bool _patchVersionBump;
|
||||
private readonly bool _majorVersionBump;
|
||||
private readonly bool _buildWillFail;
|
||||
private readonly bool _testsWillFail;
|
||||
private readonly bool _allVerificationsPassed;
|
||||
private readonly bool _hasBreakingChanges;
|
||||
|
||||
public StubRemediationPlanner(
|
||||
bool patchVersionBump = false,
|
||||
bool majorVersionBump = false,
|
||||
bool buildWillFail = false,
|
||||
bool testsWillFail = false,
|
||||
bool allVerificationsPassed = false,
|
||||
bool hasBreakingChanges = false)
|
||||
{
|
||||
_patchVersionBump = patchVersionBump;
|
||||
_majorVersionBump = majorVersionBump;
|
||||
_buildWillFail = buildWillFail;
|
||||
_testsWillFail = testsWillFail;
|
||||
_allVerificationsPassed = allVerificationsPassed;
|
||||
_hasBreakingChanges = hasBreakingChanges;
|
||||
}
|
||||
|
||||
public Task<RemediationPlan> GeneratePlanAsync(
|
||||
RemediationPlanRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var planId = $"plan-{Guid.NewGuid():N}";
|
||||
|
||||
var (actionType, risk, confidence) = DetermineStepDetails(request);
|
||||
|
||||
var steps = new List<RemediationStep>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Order = 1,
|
||||
ActionType = actionType,
|
||||
FilePath = GetFilePath(request),
|
||||
Description = $"Fix {request.VulnerabilityId}",
|
||||
PreviousValue = "old",
|
||||
NewValue = "new",
|
||||
Risk = risk
|
||||
}
|
||||
};
|
||||
|
||||
var authority = DetermineAuthority(request);
|
||||
var prReady = authority == RemediationAuthority.Verified;
|
||||
var notReadyReason = GetNotReadyReason();
|
||||
|
||||
if (_hasBreakingChanges)
|
||||
{
|
||||
confidence *= 0.6;
|
||||
}
|
||||
|
||||
var plan = new RemediationPlan
|
||||
{
|
||||
PlanId = planId,
|
||||
Request = request,
|
||||
Steps = steps,
|
||||
ExpectedDelta = new ExpectedSbomDelta
|
||||
{
|
||||
Added = Array.Empty<string>(),
|
||||
Removed = Array.Empty<string>(),
|
||||
Upgraded = new Dictionary<string, string>
|
||||
{
|
||||
[request.ComponentPurl] = request.ComponentPurl + "-fixed"
|
||||
},
|
||||
NetVulnerabilityChange = -1
|
||||
},
|
||||
RiskAssessment = risk,
|
||||
TestRequirements = new RemediationTestRequirements
|
||||
{
|
||||
TestSuites = new[] { "unit", "integration" },
|
||||
MinCoverage = 80,
|
||||
RequireAllPass = true,
|
||||
Timeout = TimeSpan.FromMinutes(15)
|
||||
},
|
||||
Authority = authority,
|
||||
PrReady = prReady,
|
||||
NotReadyReason = notReadyReason,
|
||||
ConfidenceScore = confidence,
|
||||
ModelId = "stub-model",
|
||||
GeneratedAt = DateTime.UtcNow.ToString("o"),
|
||||
InputHashes = new[] { $"input:{request.FindingId}", $"input:{request.ArtifactDigest}" },
|
||||
EvidenceRefs = new[] { "evidence/ref-001" }
|
||||
};
|
||||
|
||||
_plans[planId] = plan;
|
||||
return Task.FromResult(plan);
|
||||
}
|
||||
|
||||
private (string ActionType, RemediationRisk Risk, double Confidence) DetermineStepDetails(
|
||||
RemediationPlanRequest request)
|
||||
{
|
||||
var actionType = request.RemediationType switch
|
||||
{
|
||||
RemediationType.Bump => "update_package",
|
||||
RemediationType.Upgrade => "update_base_image",
|
||||
RemediationType.Config => "update_config",
|
||||
RemediationType.Backport => "apply_patch",
|
||||
_ => "update_package"
|
||||
};
|
||||
|
||||
if (_patchVersionBump)
|
||||
return (actionType, RemediationRisk.Low, 0.95);
|
||||
|
||||
if (_majorVersionBump)
|
||||
return (actionType, RemediationRisk.High, 0.65);
|
||||
|
||||
return (actionType, RemediationRisk.Medium, 0.85);
|
||||
}
|
||||
|
||||
private string GetFilePath(RemediationPlanRequest request)
|
||||
{
|
||||
if (request.ComponentPurl.StartsWith("pkg:npm"))
|
||||
return "package.json";
|
||||
if (request.ComponentPurl.StartsWith("pkg:pypi"))
|
||||
return "requirements.txt";
|
||||
if (request.ComponentPurl.StartsWith("pkg:oci"))
|
||||
return "Dockerfile";
|
||||
return "package.json";
|
||||
}
|
||||
|
||||
private RemediationAuthority DetermineAuthority(RemediationPlanRequest request)
|
||||
{
|
||||
if (!request.AutoCreatePr)
|
||||
return RemediationAuthority.Draft;
|
||||
|
||||
if (_buildWillFail || _testsWillFail)
|
||||
return RemediationAuthority.Suggestion;
|
||||
|
||||
if (_allVerificationsPassed)
|
||||
return RemediationAuthority.Verified;
|
||||
|
||||
return RemediationAuthority.Draft;
|
||||
}
|
||||
|
||||
private string? GetNotReadyReason()
|
||||
{
|
||||
if (_buildWillFail)
|
||||
return "Build failed during verification";
|
||||
if (_testsWillFail)
|
||||
return "Tests failed during verification";
|
||||
return null;
|
||||
}
|
||||
|
||||
public Task<bool> ValidatePlanAsync(string planId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
return Task.FromResult(_plans.ContainsKey(planId));
|
||||
}
|
||||
|
||||
public Task<RemediationPlan?> GetPlanAsync(string planId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_plans.TryGetValue(planId, out var plan);
|
||||
return Task.FromResult(plan);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubPullRequestGenerator : IPullRequestGenerator
|
||||
{
|
||||
private readonly Dictionary<string, PullRequestResult> _prs = new();
|
||||
private int _prCounter;
|
||||
|
||||
public string ScmType => ScmTypeOverride ?? "github";
|
||||
public string? ScmTypeOverride { get; set; }
|
||||
|
||||
public Task<PullRequestResult> CreatePullRequestAsync(
|
||||
RemediationPlan plan,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var prId = $"pr-{Guid.NewGuid():N}";
|
||||
_prCounter++;
|
||||
|
||||
var branchName = $"stellaops-fix-{plan.Request.VulnerabilityId.ToLowerInvariant()}-{_prCounter}";
|
||||
|
||||
var result = new PullRequestResult
|
||||
{
|
||||
PrId = prId,
|
||||
PrNumber = _prCounter,
|
||||
Url = $"https://github.com/test/repo/pull/{_prCounter}",
|
||||
BranchName = branchName,
|
||||
Status = PullRequestStatus.Open,
|
||||
CreatedAt = DateTime.UtcNow.ToString("o"),
|
||||
UpdatedAt = DateTime.UtcNow.ToString("o")
|
||||
};
|
||||
|
||||
_prs[prId] = result;
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public Task<PullRequestResult> GetStatusAsync(string prId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_prs.TryGetValue(prId, out var result))
|
||||
throw new InvalidOperationException($"PR {prId} not found");
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
public Task UpdateWithDeltaVerdictAsync(
|
||||
string prId,
|
||||
DeltaVerdictResult deltaVerdict,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_prs.TryGetValue(prId, out var result))
|
||||
throw new InvalidOperationException($"PR {prId} not found");
|
||||
|
||||
_prs[prId] = result with
|
||||
{
|
||||
DeltaVerdict = deltaVerdict,
|
||||
UpdatedAt = DateTime.UtcNow.ToString("o")
|
||||
};
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task ClosePullRequestAsync(string prId, string reason, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_prs.TryGetValue(prId, out var result))
|
||||
throw new InvalidOperationException($"PR {prId} not found");
|
||||
|
||||
_prs[prId] = result with
|
||||
{
|
||||
Status = PullRequestStatus.Closed,
|
||||
StatusMessage = reason,
|
||||
UpdatedAt = DateTime.UtcNow.ToString("o")
|
||||
};
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -10,11 +10,13 @@ using Microsoft.Extensions.Options;
|
||||
using StellaOps.AdvisoryAI.Providers;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class SbomContextHttpClientTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GetContextAsync_MapsPayloadToDocument()
|
||||
{
|
||||
const string payload = """
|
||||
@@ -98,7 +100,8 @@ public sealed class SbomContextHttpClientTests
|
||||
Assert.Contains("includeBlastRadius=true", handler.LastRequest.RequestUri!.Query);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GetContextAsync_ReturnsNullOnNotFound()
|
||||
{
|
||||
var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.NotFound));
|
||||
@@ -110,7 +113,8 @@ public sealed class SbomContextHttpClientTests
|
||||
Assert.Null(result);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task GetContextAsync_ThrowsForServerError()
|
||||
{
|
||||
var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError)
|
||||
|
||||
@@ -2,11 +2,13 @@ using FluentAssertions;
|
||||
using StellaOps.AdvisoryAI.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class SbomContextRequestTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void Constructor_NormalizesWhitespaceAndLimits()
|
||||
{
|
||||
var request = new SbomContextRequest(
|
||||
@@ -25,7 +27,8 @@ public sealed class SbomContextRequestTests
|
||||
request.IncludeBlastRadius.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void Constructor_AllowsNullPurlAndDefaults()
|
||||
{
|
||||
var request = new SbomContextRequest(artifactId: "scan-123", purl: null);
|
||||
|
||||
@@ -12,11 +12,13 @@ using StellaOps.AdvisoryAI.Providers;
|
||||
using StellaOps.AdvisoryAI.Retrievers;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class SbomContextRetrieverTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RetrieveAsync_ReturnsDeterministicOrderingAndMetadata()
|
||||
{
|
||||
var document = new SbomContextDocument(
|
||||
@@ -103,7 +105,8 @@ public sealed class SbomContextRetrieverTests
|
||||
result.Metadata["blast_radius_present"].Should().Be(bool.TrueString);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RetrieveAsync_ReturnsEmptyWhenNoDocument()
|
||||
{
|
||||
var client = new FakeSbomContextClient(null);
|
||||
@@ -119,7 +122,8 @@ public sealed class SbomContextRetrieverTests
|
||||
result.BlastRadius.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RetrieveAsync_HonorsEnvironmentFlagToggle()
|
||||
{
|
||||
var document = new SbomContextDocument(
|
||||
@@ -152,7 +156,8 @@ public sealed class SbomContextRetrieverTests
|
||||
client.LastQuery.IncludeBlastRadius.Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public async Task RetrieveAsync_DeduplicatesDependencyPaths()
|
||||
{
|
||||
var duplicatePath = ImmutableArray.Create(
|
||||
|
||||
@@ -2,11 +2,13 @@ using FluentAssertions;
|
||||
using StellaOps.AdvisoryAI.Tools;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class SemanticVersionTests
|
||||
{
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("1.2.3", 1, 2, 3, false)]
|
||||
[InlineData("1.2.3-alpha", 1, 2, 3, true)]
|
||||
[InlineData("0.0.1+build", 0, 0, 1, false)]
|
||||
@@ -21,7 +23,8 @@ public sealed class SemanticVersionTests
|
||||
(version.PreRelease.Count > 0).Should().Be(hasPreRelease);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("01.0.0")]
|
||||
[InlineData("1..0")]
|
||||
[InlineData("1.0.0-")]
|
||||
@@ -33,7 +36,8 @@ public sealed class SemanticVersionTests
|
||||
act.Should().Throw<FormatException>();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("1.2.3", "1.2.3", 0)]
|
||||
[InlineData("1.2.3", "1.2.4", -1)]
|
||||
[InlineData("1.3.0", "1.2.9", 1)]
|
||||
@@ -48,7 +52,8 @@ public sealed class SemanticVersionTests
|
||||
Math.Sign(leftVersion.CompareTo(rightVersion)).Should().Be(expectedSign);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Theory]
|
||||
[InlineData("1.2.3", ">=1.0.0,<2.0.0", true)]
|
||||
[InlineData("0.9.0", ">=1.0.0", false)]
|
||||
[InlineData("1.2.3-beta", ">=1.2.3", false)]
|
||||
@@ -61,7 +66,8 @@ public sealed class SemanticVersionTests
|
||||
SemanticVersionRange.Satisfies(version, range).Should().Be(expected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void DeterministicToolset_ComparesSemverAndEvr()
|
||||
{
|
||||
IDeterministicToolset toolset = new DeterministicToolset();
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
<ProjectReference Include="..\..\..\Concelier\__Libraries\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj" />
|
||||
<ProjectReference Include="..\..\..\Concelier\__Libraries\StellaOps.Concelier.RawModels\StellaOps.Concelier.RawModels.csproj" />
|
||||
<ProjectReference Include="..\..\..\Excititor\__Libraries\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
|
||||
<ProjectReference Include="..\..\..\__Libraries\StellaOps.TestKit\StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="TestData/*.json">
|
||||
|
||||
@@ -11,11 +11,13 @@ using StellaOps.AdvisoryAI.Abstractions;
|
||||
using StellaOps.AdvisoryAI.Documents;
|
||||
using Xunit;
|
||||
|
||||
using StellaOps.TestKit;
|
||||
namespace StellaOps.AdvisoryAI.Tests;
|
||||
|
||||
public sealed class ToolsetServiceCollectionExtensionsTests
|
||||
{
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void AddAdvisoryDeterministicToolset_RegistersSingleton()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
@@ -29,7 +31,8 @@ public sealed class ToolsetServiceCollectionExtensionsTests
|
||||
Assert.Same(toolsetA, toolsetB);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Trait("Category", TestCategories.Unit)]
|
||||
[Fact]
|
||||
public void AddAdvisoryPipeline_RegistersOrchestrator()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
|
||||
Reference in New Issue
Block a user