Refactor code structure and optimize performance across multiple modules

This commit is contained in:
StellaOps Bot
2025-12-26 20:03:22 +02:00
parent c786faae84
commit b4fc66feb6
3353 changed files with 88254 additions and 1590657 deletions

View File

@@ -15,6 +15,8 @@ using StellaOps.AdvisoryAI.Orchestration;
using StellaOps.AdvisoryAI.Prompting;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryGuardrailInjectionTests
@@ -35,7 +37,8 @@ public sealed class AdvisoryGuardrailInjectionTests
public static IEnumerable<object[]> InjectionPayloads =>
HarnessCases.Value.Select(testCase => new object[] { testCase });
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[MemberData(nameof(InjectionPayloads))]
public async Task EvaluateAsync_CompliesWithGuardrailHarness(InjectionCase testCase)
{

View File

@@ -12,11 +12,14 @@ using StellaOps.AdvisoryAI.Guardrails;
using StellaOps.AdvisoryAI.Hosting;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryGuardrailOptionsBindingTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task AddAdvisoryAiCore_ConfiguresGuardrailOptionsFromServiceOptions()
{
var tempRoot = CreateTempDirectory();
@@ -47,7 +50,8 @@ public sealed class AdvisoryGuardrailOptionsBindingTests
options.BlockedPhrases.Should().Contain("dump cache");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task AddAdvisoryAiCore_ThrowsWhenPhraseFileMissing()
{
var tempRoot = CreateTempDirectory();

View File

@@ -16,6 +16,8 @@ using StellaOps.AdvisoryAI.Orchestration;
using StellaOps.AdvisoryAI.Prompting;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryGuardrailPerformanceTests
@@ -27,7 +29,8 @@ public sealed class AdvisoryGuardrailPerformanceTests
public static IEnumerable<object[]> PerfScenarios => LoadPerfScenarios();
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[MemberData(nameof(PerfScenarios))]
public async Task EvaluateAsync_CompletesWithinBudget(PerfScenario scenario)
{
@@ -53,7 +56,8 @@ public sealed class AdvisoryGuardrailPerformanceTests
$"{scenario.Name} exceeded the allotted {scenario.MaxDurationMs} ms budget (measured {stopwatch.ElapsedMilliseconds} ms)");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task EvaluateAsync_HonorsSeededBlockedPhrases()
{
var phrases = LoadSeededBlockedPhrases();

View File

@@ -7,11 +7,13 @@ using StellaOps.AdvisoryAI.Orchestration;
using StellaOps.AdvisoryAI.Prompting;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryGuardrailPipelineTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task EvaluateAsync_BlocksWhenCitationsMissing()
{
var options = Options.Create(new AdvisoryGuardrailOptions { RequireCitations = true });
@@ -31,7 +33,8 @@ public sealed class AdvisoryGuardrailPipelineTests
Assert.Contains(result.Violations, violation => violation.Code == "citation_missing");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task EvaluateAsync_RedactsSecrets()
{
var options = Options.Create(new AdvisoryGuardrailOptions());

View File

@@ -17,13 +17,16 @@ using StellaOps.AdvisoryAI.Tools;
using StellaOps.AdvisoryAI.Inference;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryPipelineExecutorTests : IDisposable
{
private readonly StubMeterFactory _meterFactory = new();
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ExecuteAsync_SavesOutputAndProvenance()
{
var plan = BuildMinimalPlan(cacheKey: "CACHE-1");
@@ -49,7 +52,8 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
saved.Guardrail.Metadata.Should().ContainKey("prompt_length");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ExecuteAsync_PersistsGuardrailOutcome()
{
var plan = BuildMinimalPlan(cacheKey: "CACHE-2");
@@ -71,7 +75,8 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
saved.Prompt.Should().Be("{\"prompt\":\"value\"}");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ExecuteAsync_RecordsTelemetryMeasurements()
{
using var listener = new MeterListener();
@@ -124,7 +129,8 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
Math.Abs(measurement.Value - 1d) < 0.0001);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ExecuteAsync_ComputesPartialCitationCoverage()
{
using var listener = new MeterListener();
@@ -163,7 +169,8 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
Math.Abs(measurement.Value - 0.5d) < 0.0001);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ExecuteAsync_RecordsInferenceMetadata()
{
var plan = BuildMinimalPlan(cacheKey: "CACHE-4");

View File

@@ -13,11 +13,13 @@ using StellaOps.AdvisoryAI.Orchestration;
using StellaOps.AdvisoryAI.Tools;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryPipelineOrchestratorTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePlanAsync_ComposesDeterministicPlan()
{
var structuredRetriever = new FakeStructuredRetriever();
@@ -63,7 +65,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
Assert.Equal(plan.CacheKey, secondPlan.CacheKey);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePlanAsync_RemainsDeterministicAcrossMultipleRuns()
{
var structuredRetriever = new ShufflingStructuredRetriever();
@@ -116,7 +119,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
}
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePlanAsync_PopulatesMetadataCountsFromEvidence()
{
var structuredRetriever = new FakeStructuredRetriever();
@@ -156,7 +160,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
metadata["sbom_blast_impacted_workloads"].Should().Be("3");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePlanAsync_WhenArtifactIdMissing_SkipsSbomContext()
{
var structuredRetriever = new FakeStructuredRetriever();
@@ -188,7 +193,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
Assert.DoesNotContain("sbom_dependency_path_count", plan.Metadata.Keys);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePlanAsync_RespectsOptionFlagsAndProducesStableCacheKey()
{
var structuredRetriever = new FakeStructuredRetriever();
@@ -227,7 +233,8 @@ public sealed class AdvisoryPipelineOrchestratorTests
Assert.DoesNotContain(planOne.Metadata.Keys, key => key.StartsWith("sbom_blast_", StringComparison.Ordinal));
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePlanAsync_RemainsDeterministicWhenRetrieverOrderChanges()
{
var structuredRetriever = new ShufflingStructuredRetriever();

View File

@@ -10,11 +10,13 @@ using StellaOps.AdvisoryAI.Orchestration;
using StellaOps.AdvisoryAI.Tools;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryPipelinePlanResponseTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public void FromPlan_ProjectsMetadataAndCounts()
{
var request = new AdvisoryTaskRequest(AdvisoryTaskType.Summary, "adv-key");

View File

@@ -15,11 +15,13 @@ using StellaOps.AdvisoryAI.Tools;
using StellaOps.AdvisoryAI.Tests.TestUtilities;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryPlanCacheTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SetAndRetrieve_ReturnsCachedPlan()
{
var timeProvider = new DeterministicTimeProvider(DateTimeOffset.UtcNow);
@@ -34,7 +36,8 @@ public sealed class AdvisoryPlanCacheTests
retrieved.Metadata.Should().ContainKey("task_type");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ExpiredEntries_AreEvicted()
{
var start = DateTimeOffset.UtcNow;
@@ -49,7 +52,8 @@ public sealed class AdvisoryPlanCacheTests
retrieved.Should().BeNull();
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SetAsync_ReplacesPlanAndRefreshesExpiration()
{
var timeProvider = new DeterministicTimeProvider(DateTimeOffset.UtcNow);
@@ -69,7 +73,8 @@ public sealed class AdvisoryPlanCacheTests
retrieved!.Request.AdvisoryKey.Should().Be("ADV-999");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SetAsync_WithInterleavedKeysRemainsDeterministic()
{
var timeProvider = new DeterministicTimeProvider(DateTimeOffset.UtcNow);
@@ -108,7 +113,8 @@ public sealed class AdvisoryPlanCacheTests
}
}
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData(7)]
[InlineData(42)]
[InlineData(512)]

View File

@@ -14,6 +14,8 @@ using StellaOps.AdvisoryAI.Tools;
using Xunit;
using Xunit.Abstractions;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryPromptAssemblerTests
@@ -25,7 +27,8 @@ public sealed class AdvisoryPromptAssemblerTests
_output = output;
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task AssembleAsync_ProducesDeterministicPrompt()
{
var plan = BuildPlan();
@@ -43,7 +46,8 @@ public sealed class AdvisoryPromptAssemblerTests
await AssertPromptMatchesGoldenAsync("summary-prompt.json", prompt.Prompt);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task AssembleAsync_ProducesConflictPromptGolden()
{
var plan = BuildPlan(AdvisoryTaskType.Conflict);
@@ -56,7 +60,8 @@ public sealed class AdvisoryPromptAssemblerTests
prompt.Metadata["task_type"].Should().Be(nameof(AdvisoryTaskType.Conflict));
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task AssembleAsync_TruncatesVectorPreviewsToMaintainPromptSize()
{
var longPreview = new string('A', 700);

View File

@@ -7,11 +7,13 @@ using StellaOps.AdvisoryAI.Documents;
using StellaOps.AdvisoryAI.Retrievers;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryStructuredRetrieverTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RetrieveAsync_ReturnsCsafChunksWithMetadata()
{
var provider = CreateProvider(
@@ -33,7 +35,8 @@ public sealed class AdvisoryStructuredRetrieverTests
result.Chunks.Any(c => c.Section == "document.notes").Should().BeTrue();
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RetrieveAsync_ReturnsOsvChunksWithAffectedMetadata()
{
var provider = CreateProvider(
@@ -53,7 +56,8 @@ public sealed class AdvisoryStructuredRetrieverTests
result.Chunks.First(c => c.Section.StartsWith("affected", StringComparison.OrdinalIgnoreCase)).Metadata.Should().ContainKey("package");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RetrieveAsync_ReturnsOpenVexChunksWithStatusMetadata()
{
var provider = CreateProvider(
@@ -73,7 +77,8 @@ public sealed class AdvisoryStructuredRetrieverTests
result.Chunks.Should().AllSatisfy(chunk => chunk.Section.Should().Be("vex.statements"));
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RetrieveAsync_FiltersToPreferredSections()
{
var provider = CreateProvider(

View File

@@ -7,11 +7,13 @@ using StellaOps.AdvisoryAI.Orchestration;
using StellaOps.AdvisoryAI.Queue;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryTaskQueueTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task EnqueueAndDequeue_ReturnsMessageInOrder()
{
var options = Options.Create(new AdvisoryTaskQueueOptions { Capacity = 10, DequeueWaitInterval = TimeSpan.FromMilliseconds(50) });

View File

@@ -7,11 +7,13 @@ using StellaOps.AdvisoryAI.Tests.TestUtilities;
using StellaOps.AdvisoryAI.Vectorization;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class AdvisoryVectorRetrieverTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SearchAsync_ReturnsBestMatchingChunk()
{
var advisoryContent = """

View File

@@ -8,11 +8,13 @@ using StellaOps.Concelier.Core.Raw;
using StellaOps.Concelier.RawModels;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class ConcelierAdvisoryDocumentProviderTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GetDocumentsAsync_ReturnsMappedDocuments()
{
var rawDocument = RawDocumentFactory.CreateAdvisory(

View File

@@ -5,11 +5,13 @@ using StellaOps.AdvisoryAI.Context;
using StellaOps.AdvisoryAI.Tools;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class DeterministicToolsetTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public void AnalyzeDependencies_ComputesRuntimeAndDevelopmentCounts()
{
var context = SbomContextResult.Create(
@@ -52,7 +54,8 @@ public sealed class DeterministicToolsetTests
libB.DevelopmentOccurrences.Should().Be(1);
}
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("semver", "1.2.3", "1.2.4", -1)]
[InlineData("semver", "1.2.3", "1.2.3", 0)]
[InlineData("semver", "1.2.4", "1.2.3", 1)]
@@ -66,7 +69,8 @@ public sealed class DeterministicToolsetTests
comparison.Should().Be(expected);
}
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("semver", "1.2.3", ">=1.0.0 <2.0.0")]
[InlineData("semver", "2.0.0", ">=2.0.0")]
[InlineData("evr", "0:1.2-3", ">=0:1.0-0 <0:2.0-0")]

View File

@@ -9,11 +9,13 @@ using StellaOps.Excititor.Core;
using StellaOps.Excititor.Core.Observations;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class ExcititorVexDocumentProviderTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GetDocumentsAsync_ReturnsMappedObservation()
{
const string vulnerabilityId = "CVE-2024-9999";

View File

@@ -0,0 +1,542 @@
using System.Security.Cryptography;
using System.Text;
using FluentAssertions;
using StellaOps.AdvisoryAI.Explanation;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
/// <summary>
/// Integration tests for explanation generation with mocked LLM and evidence anchoring validation.
/// Sprint: SPRINT_20251226_015_AI_zastava_companion
/// Task: ZASTAVA-19
/// </summary>
public sealed class ExplanationGeneratorIntegrationTests
{
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_WithFullEvidence_ProducesEvidenceBackedExplanation()
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "This is a test explanation with [citation:ev-001] and [citation:ev-002].",
confidence: 0.95);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Full);
// Act
var result = await generator.GenerateAsync(request);
// Assert
result.Should().NotBeNull();
result.ExplanationId.Should().StartWith("sha256:");
result.Authority.Should().Be(ExplanationAuthority.EvidenceBacked);
result.CitationRate.Should().BeGreaterOrEqualTo(0.8);
result.Citations.Should().NotBeEmpty();
result.EvidenceRefs.Should().NotBeEmpty();
result.InputHashes.Should().HaveCount(3);
result.OutputHash.Should().NotBeNullOrEmpty();
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_WithLowCitationRate_ProducesSuggestionExplanation()
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateMinimalEvidenceContext());
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "This is a speculative explanation without proper citations.",
confidence: 0.6);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.3);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Why);
// Act
var result = await generator.GenerateAsync(request);
// Assert
result.Authority.Should().Be(ExplanationAuthority.Suggestion);
result.CitationRate.Should().BeLessThan(0.8);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_StoresResultForReplay()
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "Stored explanation [citation:ev-001].",
confidence: 0.9);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.85);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.What);
// Act
var result = await generator.GenerateAsync(request);
// Assert
var stored = await store.GetAsync(result.ExplanationId, CancellationToken.None);
stored.Should().NotBeNull();
stored!.ExplanationId.Should().Be(result.ExplanationId);
stored.OutputHash.Should().Be(result.OutputHash);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_ComputesConsistentInputHashes()
{
// Arrange
var evidenceContext = CreateFullEvidenceContext();
var evidenceService = new StubEvidenceRetrievalService(evidenceContext);
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "Consistent explanation.",
confidence: 0.88);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.85);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Evidence);
// Act
var result1 = await generator.GenerateAsync(request);
var result2 = await generator.GenerateAsync(request);
// Assert - same inputs should produce same input hashes
result1.InputHashes.Should().BeEquivalentTo(result2.InputHashes);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_ProducesValidExplanationId()
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "Test explanation content.",
confidence: 0.9);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Full);
// Act
var result = await generator.GenerateAsync(request);
// Assert
result.ExplanationId.Should().StartWith("sha256:");
result.ExplanationId.Length.Should().Be(7 + 64); // "sha256:" + 64 hex chars
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_IncludesAllEvidenceRefs()
{
// Arrange
var evidenceContext = CreateFullEvidenceContext();
var evidenceService = new StubEvidenceRetrievalService(evidenceContext);
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "Explanation with evidence.",
confidence: 0.9);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Full);
// Act
var result = await generator.GenerateAsync(request);
// Assert
var allEvidenceIds = evidenceContext.AllEvidence.Select(e => e.Id).ToList();
result.EvidenceRefs.Should().BeEquivalentTo(allEvidenceIds);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_RecordsModelIdAndTemplateVersion()
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
var promptService = new StubExplanationPromptService(templateVersion: "explain-v2.1");
var inferenceClient = new StubExplanationInferenceClient(
content: "Test.",
confidence: 0.9,
modelId: "claude:claude-3-opus:20240229");
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Full);
// Act
var result = await generator.GenerateAsync(request);
// Assert
result.ModelId.Should().Be("claude:claude-3-opus:20240229");
result.PromptTemplateVersion.Should().Be("explain-v2.1");
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_GeneratesValidSummary()
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "Detailed explanation content.",
confidence: 0.9);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Full);
// Act
var result = await generator.GenerateAsync(request);
// Assert
result.Summary.Should().NotBeNull();
result.Summary.Line1.Should().NotBeNullOrEmpty();
result.Summary.Line2.Should().NotBeNullOrEmpty();
result.Summary.Line3.Should().NotBeNullOrEmpty();
}
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData(ExplanationType.What)]
[InlineData(ExplanationType.Why)]
[InlineData(ExplanationType.Evidence)]
[InlineData(ExplanationType.Counterfactual)]
[InlineData(ExplanationType.Full)]
public async Task GenerateAsync_HandlesAllExplanationTypes(ExplanationType type)
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext());
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: $"Explanation for {type}.",
confidence: 0.9);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.85);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(type);
// Act
var result = await generator.GenerateAsync(request);
// Assert
result.Should().NotBeNull();
result.Content.Should().Contain(type.ToString());
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ValidateAsync_ReturnsTrueForValidEvidence()
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext(), validateResult: true);
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "Test.",
confidence: 0.9);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Full);
var result = await generator.GenerateAsync(request);
// Act
var isValid = await generator.ValidateAsync(result);
// Assert
isValid.Should().BeTrue();
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ValidateAsync_ReturnsFalseWhenEvidenceChanged()
{
// Arrange
var evidenceService = new StubEvidenceRetrievalService(CreateFullEvidenceContext(), validateResult: false);
var promptService = new StubExplanationPromptService();
var inferenceClient = new StubExplanationInferenceClient(
content: "Test.",
confidence: 0.9);
var citationExtractor = new StubCitationExtractor(verifiedRate: 0.9);
var store = new InMemoryExplanationStore();
var generator = new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
var request = CreateExplanationRequest(ExplanationType.Full);
var result = await generator.GenerateAsync(request);
// Act
var isValid = await generator.ValidateAsync(result);
// Assert
isValid.Should().BeFalse();
}
#region Helper Methods
private static ExplanationRequest CreateExplanationRequest(ExplanationType type) => new()
{
FindingId = "finding-001",
ArtifactDigest = "sha256:abc123",
Scope = "image",
ScopeId = "my-image:latest",
ExplanationType = type,
VulnerabilityId = "CVE-2024-1234",
ComponentPurl = "pkg:npm/lodash@4.17.20",
PlainLanguage = false,
MaxLength = 0,
CorrelationId = "corr-001"
};
private static EvidenceContext CreateFullEvidenceContext() => new()
{
SbomEvidence =
[
new EvidenceNode
{
Id = "ev-001",
Type = "sbom",
Summary = "Component lodash@4.17.20 found in SBOM",
Content = "Package: lodash, Version: 4.17.20, License: MIT",
Source = "sbom-scan",
Confidence = 0.99,
CollectedAt = "2024-01-15T10:00:00Z"
}
],
ReachabilityEvidence =
[
new EvidenceNode
{
Id = "ev-002",
Type = "reachability",
Summary = "Vulnerable function is reachable",
Content = "Call path: main.js -> utils.js -> lodash.merge()",
Source = "static-analysis",
Confidence = 0.85,
CollectedAt = "2024-01-15T10:05:00Z"
}
],
RuntimeEvidence = [],
VexEvidence =
[
new EvidenceNode
{
Id = "ev-003",
Type = "vex",
Summary = "No vendor VEX statement",
Content = "No applicable VEX statements found",
Source = "vex-lookup",
Confidence = 0.5,
CollectedAt = "2024-01-15T10:10:00Z"
}
],
PatchEvidence = [],
ContextHash = ComputeHash("full-evidence-context")
};
private static EvidenceContext CreateMinimalEvidenceContext() => new()
{
SbomEvidence =
[
new EvidenceNode
{
Id = "ev-min-001",
Type = "sbom",
Summary = "Component found",
Content = "Package exists",
Source = "sbom",
Confidence = 0.7,
CollectedAt = "2024-01-15T10:00:00Z"
}
],
ReachabilityEvidence = [],
RuntimeEvidence = [],
VexEvidence = [],
PatchEvidence = [],
ContextHash = ComputeHash("minimal-evidence-context")
};
private static string ComputeHash(string content)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
return Convert.ToHexStringLower(bytes);
}
#endregion
#region Stub Implementations
private sealed class StubEvidenceRetrievalService : IEvidenceRetrievalService
{
private readonly EvidenceContext _context;
private readonly bool _validateResult;
public StubEvidenceRetrievalService(EvidenceContext context, bool validateResult = true)
{
_context = context;
_validateResult = validateResult;
}
public Task<EvidenceContext> RetrieveEvidenceAsync(
string findingId, string artifactDigest, string vulnerabilityId,
string? componentPurl = null, CancellationToken cancellationToken = default)
=> Task.FromResult(_context);
public Task<EvidenceNode?> GetEvidenceNodeAsync(string evidenceId, CancellationToken cancellationToken = default)
=> Task.FromResult(_context.AllEvidence.FirstOrDefault(e => e.Id == evidenceId));
public Task<bool> ValidateEvidenceAsync(IEnumerable<string> evidenceIds, CancellationToken cancellationToken = default)
=> Task.FromResult(_validateResult);
}
private sealed class StubExplanationPromptService : IExplanationPromptService
{
private readonly string _templateVersion;
public StubExplanationPromptService(string templateVersion = "explain-v1.0")
{
_templateVersion = templateVersion;
}
public Task<ExplanationPrompt> BuildPromptAsync(
ExplanationRequest request, EvidenceContext evidence, CancellationToken cancellationToken = default)
=> Task.FromResult(new ExplanationPrompt
{
Content = $"Explain {request.VulnerabilityId} for {request.ExplanationType}",
TemplateVersion = _templateVersion
});
public Task<ExplanationSummary> GenerateSummaryAsync(
string content, ExplanationType type, CancellationToken cancellationToken = default)
=> Task.FromResult(new ExplanationSummary
{
Line1 = "What: Vulnerability detected",
Line2 = "Why: Reachable code path",
Line3 = "Action: Update dependency"
});
}
private sealed class StubExplanationInferenceClient : IExplanationInferenceClient
{
private readonly string _content;
private readonly double _confidence;
private readonly string _modelId;
public StubExplanationInferenceClient(string content, double confidence, string modelId = "stub-model:v1")
{
_content = content;
_confidence = confidence;
_modelId = modelId;
}
public Task<ExplanationInferenceResult> GenerateAsync(
ExplanationPrompt prompt, CancellationToken cancellationToken = default)
=> Task.FromResult(new ExplanationInferenceResult
{
Content = _content,
Confidence = _confidence,
ModelId = _modelId
});
}
private sealed class StubCitationExtractor : ICitationExtractor
{
private readonly double _verifiedRate;
public StubCitationExtractor(double verifiedRate)
{
_verifiedRate = verifiedRate;
}
public Task<IReadOnlyList<ExplanationCitation>> ExtractCitationsAsync(
string content, EvidenceContext evidence, CancellationToken cancellationToken = default)
{
var citations = new List<ExplanationCitation>();
var evidenceList = evidence.AllEvidence.ToList();
for (int i = 0; i < evidenceList.Count; i++)
{
var ev = evidenceList[i];
citations.Add(new ExplanationCitation
{
ClaimText = $"Claim about {ev.Type}",
EvidenceId = ev.Id,
EvidenceType = ev.Type,
Verified = i < (int)(evidenceList.Count * _verifiedRate),
EvidenceExcerpt = ev.Summary
});
}
return Task.FromResult<IReadOnlyList<ExplanationCitation>>(citations);
}
}
private sealed class InMemoryExplanationStore : IExplanationStore
{
private readonly Dictionary<string, ExplanationResult> _results = new();
private readonly Dictionary<string, ExplanationRequest> _requests = new();
public Task StoreAsync(ExplanationResult result, CancellationToken cancellationToken = default)
{
_results[result.ExplanationId] = result;
return Task.CompletedTask;
}
public Task<ExplanationResult?> GetAsync(string explanationId, CancellationToken cancellationToken = default)
=> Task.FromResult(_results.GetValueOrDefault(explanationId));
public Task<ExplanationRequest?> GetRequestAsync(string explanationId, CancellationToken cancellationToken = default)
=> Task.FromResult(_requests.GetValueOrDefault(explanationId));
public void StoreRequest(string explanationId, ExplanationRequest request)
=> _requests[explanationId] = request;
}
#endregion
}

View File

@@ -0,0 +1,500 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using FluentAssertions;
using StellaOps.AdvisoryAI.Explanation;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
/// <summary>
/// Golden tests for deterministic explanation replay.
/// Verifies that replaying an explanation with the same inputs produces identical output.
/// Sprint: SPRINT_20251226_015_AI_zastava_companion
/// Task: ZASTAVA-20
/// </summary>
public sealed class ExplanationReplayGoldenTests
{
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ReplayAsync_WithSameInputs_ProducesIdenticalOutput()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request = CreateDeterministicRequest();
// Act - Generate original
var original = await generator.GenerateAsync(request);
store.StoreRequest(original.ExplanationId, request);
// Act - Replay
var replayed = await generator.ReplayAsync(original.ExplanationId);
// Assert - Output should be identical
replayed.OutputHash.Should().Be(original.OutputHash);
replayed.Content.Should().Be(original.Content);
replayed.InputHashes.Should().BeEquivalentTo(original.InputHashes);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ReplayAsync_PreservesExplanationStructure()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request = CreateDeterministicRequest();
var original = await generator.GenerateAsync(request);
store.StoreRequest(original.ExplanationId, request);
// Act
var replayed = await generator.ReplayAsync(original.ExplanationId);
// Assert
replayed.Citations.Count.Should().Be(original.Citations.Count);
replayed.EvidenceRefs.Should().BeEquivalentTo(original.EvidenceRefs);
replayed.ConfidenceScore.Should().Be(original.ConfidenceScore);
replayed.CitationRate.Should().Be(original.CitationRate);
replayed.Authority.Should().Be(original.Authority);
replayed.ModelId.Should().Be(original.ModelId);
replayed.PromptTemplateVersion.Should().Be(original.PromptTemplateVersion);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ReplayAsync_WithChangedEvidence_ThrowsException()
{
// Arrange
var originalContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateGeneratorWithChangingEvidence(store);
var request = CreateDeterministicRequest();
var original = await generator.GenerateAsync(request);
store.StoreRequest(original.ExplanationId, request);
// Mark evidence as changed
generator.MarkEvidenceAsChanged();
// Act & Assert
var act = async () => await generator.ReplayAsync(original.ExplanationId);
await act.Should().ThrowAsync<InvalidOperationException>()
.WithMessage("*evidence has changed*");
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task MultipleReplays_ProduceIdenticalResults()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request = CreateDeterministicRequest();
var original = await generator.GenerateAsync(request);
store.StoreRequest(original.ExplanationId, request);
// Act - Replay multiple times
var replay1 = await generator.ReplayAsync(original.ExplanationId);
var replay2 = await generator.ReplayAsync(original.ExplanationId);
var replay3 = await generator.ReplayAsync(original.ExplanationId);
// Assert - All should be identical
replay1.OutputHash.Should().Be(original.OutputHash);
replay2.OutputHash.Should().Be(original.OutputHash);
replay3.OutputHash.Should().Be(original.OutputHash);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task InputHashOrder_IsConsistent()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request = CreateDeterministicRequest();
// Act - Generate twice
var result1 = await generator.GenerateAsync(request);
var result2 = await generator.GenerateAsync(request);
// Assert - Input hashes should be in same order
result1.InputHashes.Should().HaveCount(3);
result2.InputHashes.Should().HaveCount(3);
for (int i = 0; i < result1.InputHashes.Count; i++)
{
result1.InputHashes[i].Should().Be(result2.InputHashes[i],
$"Input hash at index {i} should be identical");
}
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ExplanationId_IsDeterministicFromInputsAndOutput()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request = CreateDeterministicRequest();
// Act
var result1 = await generator.GenerateAsync(request);
var result2 = await generator.GenerateAsync(request);
// Assert - Same inputs + same output = same ID
result1.ExplanationId.Should().Be(result2.ExplanationId);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task DifferentInputs_ProduceDifferentIds()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request1 = CreateDeterministicRequest() with { VulnerabilityId = "CVE-2024-0001" };
var request2 = CreateDeterministicRequest() with { VulnerabilityId = "CVE-2024-0002" };
// Act
var result1 = await generator.GenerateAsync(request1);
var result2 = await generator.GenerateAsync(request2);
// Assert
result1.ExplanationId.Should().NotBe(result2.ExplanationId);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GoldenOutput_MatchesExpectedFormat()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request = CreateDeterministicRequest();
// Act
var result = await generator.GenerateAsync(request);
// Assert - Verify golden format
result.ExplanationId.Should().StartWith("sha256:");
result.OutputHash.Should().HaveLength(64); // SHA-256 hex
result.GeneratedAt.Should().MatchRegex(@"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}");
result.InputHashes.Should().AllSatisfy(h => h.Length.Should().Be(64));
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CitationVerification_IsDeterministic()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request = CreateDeterministicRequest();
// Act
var result1 = await generator.GenerateAsync(request);
var result2 = await generator.GenerateAsync(request);
// Assert - Citations should be identical in order and verification status
result1.Citations.Count.Should().Be(result2.Citations.Count);
for (int i = 0; i < result1.Citations.Count; i++)
{
result1.Citations[i].ClaimText.Should().Be(result2.Citations[i].ClaimText);
result1.Citations[i].EvidenceId.Should().Be(result2.Citations[i].EvidenceId);
result1.Citations[i].Verified.Should().Be(result2.Citations[i].Verified);
}
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SummaryGeneration_IsDeterministic()
{
// Arrange
var evidenceContext = CreateDeterministicEvidenceContext();
var store = new InMemoryExplanationStoreWithRequests();
var generator = CreateDeterministicGenerator(evidenceContext, store);
var request = CreateDeterministicRequest();
// Act
var result1 = await generator.GenerateAsync(request);
var result2 = await generator.GenerateAsync(request);
// Assert
result1.Summary.Line1.Should().Be(result2.Summary.Line1);
result1.Summary.Line2.Should().Be(result2.Summary.Line2);
result1.Summary.Line3.Should().Be(result2.Summary.Line3);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public void OutputHash_MatchesContentHash()
{
// Arrange
var content = "This is deterministic explanation content.";
var expectedHash = ComputeHash(content);
// Act
var actualHash = ComputeHash(content);
// Assert
actualHash.Should().Be(expectedHash);
actualHash.Should().HaveLength(64);
}
#region Helper Methods
private static ExplanationRequest CreateDeterministicRequest() => new()
{
FindingId = "golden-finding-001",
ArtifactDigest = "sha256:golden123abc",
Scope = "image",
ScopeId = "golden-image:v1.0.0",
ExplanationType = ExplanationType.Full,
VulnerabilityId = "CVE-2024-GOLDEN",
ComponentPurl = "pkg:npm/golden-pkg@1.0.0",
PlainLanguage = false,
MaxLength = 0,
CorrelationId = "golden-corr-001"
};
private static EvidenceContext CreateDeterministicEvidenceContext() => new()
{
SbomEvidence =
[
new EvidenceNode
{
Id = "golden-ev-001",
Type = "sbom",
Summary = "Golden component found",
Content = "Package: golden-pkg, Version: 1.0.0",
Source = "golden-sbom",
Confidence = 0.99,
CollectedAt = "2024-01-01T00:00:00Z"
}
],
ReachabilityEvidence =
[
new EvidenceNode
{
Id = "golden-ev-002",
Type = "reachability",
Summary = "Golden function reachable",
Content = "Call path: entry -> golden_func()",
Source = "golden-analysis",
Confidence = 0.95,
CollectedAt = "2024-01-01T00:00:01Z"
}
],
RuntimeEvidence = [],
VexEvidence = [],
PatchEvidence = [],
ContextHash = ComputeHash("golden-evidence-context-v1")
};
private static EvidenceAnchoredExplanationGenerator CreateDeterministicGenerator(
EvidenceContext evidenceContext,
InMemoryExplanationStoreWithRequests store)
{
var evidenceService = new DeterministicEvidenceService(evidenceContext);
var promptService = new DeterministicPromptService();
var inferenceClient = new DeterministicInferenceClient();
var citationExtractor = new DeterministicCitationExtractor();
return new EvidenceAnchoredExplanationGenerator(
evidenceService, promptService, inferenceClient, citationExtractor, store);
}
private static ChangingEvidenceGenerator CreateGeneratorWithChangingEvidence(
InMemoryExplanationStoreWithRequests store)
{
return new ChangingEvidenceGenerator(store);
}
private static string ComputeHash(string content)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
return Convert.ToHexStringLower(bytes);
}
#endregion
#region Deterministic Test Doubles
private sealed class DeterministicEvidenceService : IEvidenceRetrievalService
{
private readonly EvidenceContext _context;
public DeterministicEvidenceService(EvidenceContext context) => _context = context;
public Task<EvidenceContext> RetrieveEvidenceAsync(
string findingId, string artifactDigest, string vulnerabilityId,
string? componentPurl = null, CancellationToken cancellationToken = default)
=> Task.FromResult(_context);
public Task<EvidenceNode?> GetEvidenceNodeAsync(string evidenceId, CancellationToken cancellationToken = default)
=> Task.FromResult(_context.AllEvidence.FirstOrDefault(e => e.Id == evidenceId));
public Task<bool> ValidateEvidenceAsync(IEnumerable<string> evidenceIds, CancellationToken cancellationToken = default)
=> Task.FromResult(true);
}
private sealed class DeterministicPromptService : IExplanationPromptService
{
public Task<ExplanationPrompt> BuildPromptAsync(
ExplanationRequest request, EvidenceContext evidence, CancellationToken cancellationToken = default)
=> Task.FromResult(new ExplanationPrompt
{
Content = $"GOLDEN_PROMPT:{request.VulnerabilityId}:{evidence.ContextHash}",
TemplateVersion = "golden-template-v1.0"
});
public Task<ExplanationSummary> GenerateSummaryAsync(
string content, ExplanationType type, CancellationToken cancellationToken = default)
=> Task.FromResult(new ExplanationSummary
{
Line1 = "Golden: What happened",
Line2 = "Golden: Why it matters",
Line3 = "Golden: Next steps"
});
}
private sealed class DeterministicInferenceClient : IExplanationInferenceClient
{
public Task<ExplanationInferenceResult> GenerateAsync(
ExplanationPrompt prompt, CancellationToken cancellationToken = default)
{
// Deterministic output based on prompt hash
var content = $"GOLDEN_EXPLANATION:hash={ComputeHash(prompt.Content)}";
return Task.FromResult(new ExplanationInferenceResult
{
Content = content,
Confidence = 0.95,
ModelId = "golden-model:v1.0"
});
}
private static string ComputeHash(string content)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
return Convert.ToHexStringLower(bytes)[..16];
}
}
private sealed class DeterministicCitationExtractor : ICitationExtractor
{
public Task<IReadOnlyList<ExplanationCitation>> ExtractCitationsAsync(
string content, EvidenceContext evidence, CancellationToken cancellationToken = default)
{
// Deterministic citations based on evidence order
var citations = evidence.AllEvidence.Select((ev, i) => new ExplanationCitation
{
ClaimText = $"Golden claim {i + 1}",
EvidenceId = ev.Id,
EvidenceType = ev.Type,
Verified = true,
EvidenceExcerpt = ev.Summary
}).ToList();
return Task.FromResult<IReadOnlyList<ExplanationCitation>>(citations);
}
}
private sealed class InMemoryExplanationStoreWithRequests : IExplanationStore
{
private readonly Dictionary<string, ExplanationResult> _results = new();
private readonly Dictionary<string, ExplanationRequest> _requests = new();
public Task StoreAsync(ExplanationResult result, CancellationToken cancellationToken = default)
{
_results[result.ExplanationId] = result;
return Task.CompletedTask;
}
public Task<ExplanationResult?> GetAsync(string explanationId, CancellationToken cancellationToken = default)
=> Task.FromResult(_results.GetValueOrDefault(explanationId));
public Task<ExplanationRequest?> GetRequestAsync(string explanationId, CancellationToken cancellationToken = default)
=> Task.FromResult(_requests.GetValueOrDefault(explanationId));
public void StoreRequest(string explanationId, ExplanationRequest request)
=> _requests[explanationId] = request;
}
private sealed class ChangingEvidenceGenerator : IExplanationGenerator
{
private readonly InMemoryExplanationStoreWithRequests _store;
private bool _evidenceChanged = false;
public ChangingEvidenceGenerator(InMemoryExplanationStoreWithRequests store)
{
_store = store;
}
public void MarkEvidenceAsChanged() => _evidenceChanged = true;
public async Task<ExplanationResult> GenerateAsync(ExplanationRequest request, CancellationToken cancellationToken = default)
{
var result = new ExplanationResult
{
ExplanationId = $"sha256:{ComputeHash(JsonSerializer.Serialize(request))}",
Content = "Test content",
Summary = new ExplanationSummary { Line1 = "L1", Line2 = "L2", Line3 = "L3" },
Citations = [],
ConfidenceScore = 0.9,
CitationRate = 0.9,
Authority = ExplanationAuthority.EvidenceBacked,
EvidenceRefs = ["ev-001"],
ModelId = "test-model",
PromptTemplateVersion = "v1",
InputHashes = [ComputeHash("input")],
GeneratedAt = DateTime.UtcNow.ToString("O"),
OutputHash = ComputeHash("Test content")
};
await _store.StoreAsync(result, cancellationToken);
return result;
}
public async Task<ExplanationResult> ReplayAsync(string explanationId, CancellationToken cancellationToken = default)
{
var original = await _store.GetAsync(explanationId, cancellationToken)
?? throw new InvalidOperationException($"Explanation {explanationId} not found");
if (_evidenceChanged)
{
throw new InvalidOperationException("Input evidence has changed since original explanation");
}
return original;
}
public Task<bool> ValidateAsync(ExplanationResult result, CancellationToken cancellationToken = default)
=> Task.FromResult(!_evidenceChanged);
private static string ComputeHash(string content)
{
var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(content));
return Convert.ToHexStringLower(bytes);
}
}
#endregion
}

View File

@@ -19,13 +19,15 @@ using StellaOps.AdvisoryAI.Tools;
using StellaOps.AdvisoryAI.Tests.TestUtilities;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class FileSystemAdvisoryOutputStoreTests : IDisposable
{
private readonly TempDirectory _temp = TempDirectory.Create();
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SaveAndRetrieve_RoundTripsOutput()
{
var store = CreateStore();
@@ -41,7 +43,8 @@ public sealed class FileSystemAdvisoryOutputStoreTests : IDisposable
retrieved.Metadata["inference.model_id"].Should().Be("local.prompt-preview");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task TryGetAsync_ReturnsNullWhenFileMissing()
{
var store = CreateStore();

View File

@@ -19,13 +19,15 @@ using StellaOps.AdvisoryAI.Prompting;
using StellaOps.AdvisoryAI.Tools;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class FileSystemAdvisoryPersistenceTests : IDisposable
{
private readonly TempDirectory _tempDir = new();
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task PlanCache_PersistsPlanOnDisk()
{
var serviceOptions = Options.Create(new AdvisoryAiServiceOptions
@@ -54,7 +56,8 @@ public sealed class FileSystemAdvisoryPersistenceTests : IDisposable
reloaded.Metadata.Should().ContainKey("advisory_key").WhoseValue.Should().Be("adv-key");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task OutputStore_PersistsOutputOnDisk()
{
var serviceOptions = Options.Create(new AdvisoryAiServiceOptions

View File

@@ -16,13 +16,15 @@ using StellaOps.AdvisoryAI.Tools;
using StellaOps.AdvisoryAI.Tests.TestUtilities;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class FileSystemAdvisoryPlanCacheTests : IDisposable
{
private readonly TempDirectory _temp = TempDirectory.Create();
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SetAndRetrieve_RoundTripsPlan()
{
var cache = CreateCache();
@@ -36,7 +38,8 @@ public sealed class FileSystemAdvisoryPlanCacheTests : IDisposable
retrieved.Metadata.Should().ContainKey("task_type");
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task TryGetAsync_WhenExpired_ReturnsNull()
{
var clock = new DeterministicTimeProvider(new DateTimeOffset(2025, 11, 9, 0, 0, 0, TimeSpan.Zero));
@@ -50,7 +53,8 @@ public sealed class FileSystemAdvisoryPlanCacheTests : IDisposable
retrieved.Should().BeNull();
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task BulkSeedAsync_RemainsDeterministicAcrossInstances()
{
var clock = new DeterministicTimeProvider(new DateTimeOffset(2025, 11, 9, 0, 0, 0, TimeSpan.Zero));

View File

@@ -8,6 +8,7 @@ using StellaOps.AdvisoryAI.Orchestration;
using StellaOps.AdvisoryAI.Queue;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class FileSystemAdvisoryTaskQueueTests : IDisposable
@@ -20,7 +21,8 @@ public sealed class FileSystemAdvisoryTaskQueueTests : IDisposable
Directory.CreateDirectory(_root);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task EnqueueAndDequeue_RoundTripsMessage()
{
var options = Options.Create(new AdvisoryAiServiceOptions

View File

@@ -0,0 +1,795 @@
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using StellaOps.AdvisoryAI.Inference;
using StellaOps.AdvisoryAI.Inference.LlmProviders;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
/// <summary>
/// Integration tests for offline AI inference infrastructure.
/// Sprint: SPRINT_20251226_019_AI_offline_inference
/// Task: OFFLINE-25
/// </summary>
public sealed class OfflineInferenceIntegrationTests : IDisposable
{
private readonly string _tempPath;
private readonly InMemoryLlmInferenceCache _cache;
private readonly StubLlmProvider _stubProvider;
public OfflineInferenceIntegrationTests()
{
_tempPath = Path.Combine(Path.GetTempPath(), $"stellaops_offline_tests_{Guid.NewGuid():N}");
Directory.CreateDirectory(_tempPath);
var cacheOptions = Options.Create(new LlmInferenceCacheOptions
{
Enabled = true,
DeterministicOnly = true,
DefaultTtl = TimeSpan.FromDays(7)
});
_cache = new InMemoryLlmInferenceCache(
cacheOptions,
NullLogger<InMemoryLlmInferenceCache>.Instance);
_stubProvider = new StubLlmProvider();
}
public void Dispose()
{
_cache.Dispose();
try
{
if (Directory.Exists(_tempPath))
{
Directory.Delete(_tempPath, recursive: true);
}
}
catch
{
// Ignore cleanup errors
}
}
#region Local Inference Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CompleteAsync_WithDeterministicSettings_ReturnsDeterministicResult()
{
// Arrange
var request = new LlmCompletionRequest
{
UserPrompt = "Analyze CVE-2024-1234 for log4j",
SystemPrompt = "You are a security analyst.",
Temperature = 0,
Seed = 42,
MaxTokens = 1024
};
// Act
var result1 = await _stubProvider.CompleteAsync(request);
var result2 = await _stubProvider.CompleteAsync(request);
// Assert
Assert.True(result1.Deterministic);
Assert.True(result2.Deterministic);
Assert.Equal(result1.Content, result2.Content);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CompleteAsync_WithProviderAvailabilityCheck_ReturnsTrue()
{
// Act
var isAvailable = await _stubProvider.IsAvailableAsync();
// Assert
Assert.True(isAvailable);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CompleteStreamAsync_YieldsChunks()
{
// Arrange
var request = new LlmCompletionRequest
{
UserPrompt = "Test streaming",
Temperature = 0
};
// Act
var chunks = new List<LlmStreamChunk>();
await foreach (var chunk in _stubProvider.CompleteStreamAsync(request))
{
chunks.Add(chunk);
}
// Assert
Assert.NotEmpty(chunks);
Assert.Contains(chunks, c => c.IsFinal);
}
#endregion
#region Inference Cache Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task Cache_DeterministicRequest_CachesResult()
{
// Arrange
var request = new LlmCompletionRequest
{
UserPrompt = "Cached prompt",
Temperature = 0,
Seed = 42
};
var result = new LlmCompletionResult
{
Content = "Cached response",
ModelId = "test-model",
ProviderId = "stub",
Deterministic = true,
OutputTokens = 10
};
// Act
await _cache.SetAsync(request, "stub", result);
var cached = await _cache.TryGetAsync(request, "stub");
// Assert
Assert.NotNull(cached);
Assert.Equal(result.Content, cached.Content);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task Cache_NonDeterministicRequest_DoesNotCache()
{
// Arrange
var options = Options.Create(new LlmInferenceCacheOptions
{
Enabled = true,
DeterministicOnly = true
});
using var cache = new InMemoryLlmInferenceCache(
options, NullLogger<InMemoryLlmInferenceCache>.Instance);
var request = new LlmCompletionRequest
{
UserPrompt = "Non-deterministic",
Temperature = 0.7 // Non-deterministic
};
var result = new LlmCompletionResult
{
Content = "Response",
ModelId = "test-model",
ProviderId = "stub",
Deterministic = false
};
// Act
await cache.SetAsync(request, "stub", result);
var cached = await cache.TryGetAsync(request, "stub");
// Assert
Assert.Null(cached);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task Cache_SameInputsDifferentSeeds_SeparateCacheEntries()
{
// Arrange
var request1 = new LlmCompletionRequest
{
UserPrompt = "Test prompt",
Temperature = 0,
Seed = 42
};
var request2 = new LlmCompletionRequest
{
UserPrompt = "Test prompt",
Temperature = 0,
Seed = 123
};
var result1 = new LlmCompletionResult
{
Content = "Response with seed 42",
ModelId = "test-model",
ProviderId = "stub",
Deterministic = true
};
var result2 = new LlmCompletionResult
{
Content = "Response with seed 123",
ModelId = "test-model",
ProviderId = "stub",
Deterministic = true
};
// Act
await _cache.SetAsync(request1, "stub", result1);
await _cache.SetAsync(request2, "stub", result2);
var cached1 = await _cache.TryGetAsync(request1, "stub");
var cached2 = await _cache.TryGetAsync(request2, "stub");
// Assert
Assert.NotNull(cached1);
Assert.NotNull(cached2);
Assert.NotEqual(cached1.Content, cached2.Content);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public void Cache_Statistics_TracksHitsAndMisses()
{
// Act
var stats = _cache.GetStatistics();
// Assert
Assert.NotNull(stats);
Assert.Equal(0, stats.Hits);
Assert.True(stats.HitRate >= 0 && stats.HitRate <= 1);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CachingLlmProvider_UsesCache()
{
// Arrange
var countingProvider = new CallCountingLlmProvider();
var cachingProvider = new CachingLlmProvider(
countingProvider,
_cache,
NullLogger<CachingLlmProvider>.Instance);
var request = new LlmCompletionRequest
{
UserPrompt = "Test caching",
Temperature = 0,
Seed = 42
};
// Act - First call hits provider
var result1 = await cachingProvider.CompleteAsync(request);
Assert.Equal(1, countingProvider.CallCount);
// Act - Second call should use cache
var result2 = await cachingProvider.CompleteAsync(request);
Assert.Equal(1, countingProvider.CallCount); // Still 1, used cache
// Assert
Assert.Equal(result1.Content, result2.Content);
}
#endregion
#region Bundle Verification Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task BundleManager_VerifyBundle_ValidBundle_ReturnsValid()
{
// Arrange
var bundlePath = Path.Combine(_tempPath, "valid-bundle");
CreateValidBundle(bundlePath);
var manager = new FileSystemModelBundleManager(_tempPath);
// Act
var result = await manager.VerifyBundleAsync(bundlePath);
// Assert
Assert.True(result.Valid);
Assert.Empty(result.FailedFiles);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task BundleManager_VerifyBundle_MissingManifest_ReturnsInvalid()
{
// Arrange
var bundlePath = Path.Combine(_tempPath, "no-manifest");
Directory.CreateDirectory(bundlePath);
var manager = new FileSystemModelBundleManager(_tempPath);
// Act
var result = await manager.VerifyBundleAsync(bundlePath);
// Assert
Assert.False(result.Valid);
Assert.NotNull(result.ErrorMessage);
Assert.Contains("manifest.json", result.ErrorMessage);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task BundleManager_VerifyBundle_CorruptedFile_ReturnsInvalid()
{
// Arrange
var bundlePath = Path.Combine(_tempPath, "corrupted-bundle");
CreateValidBundle(bundlePath);
// Corrupt a file
var modelFile = Path.Combine(bundlePath, "model.gguf");
await File.WriteAllTextAsync(modelFile, "corrupted data");
var manager = new FileSystemModelBundleManager(_tempPath);
// Act
var result = await manager.VerifyBundleAsync(bundlePath);
// Assert
Assert.False(result.Valid);
Assert.NotEmpty(result.FailedFiles);
Assert.Contains(result.FailedFiles, f => f.Contains("model.gguf"));
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task BundleManager_VerifyBundle_MissingFile_ReturnsInvalid()
{
// Arrange
var bundlePath = Path.Combine(_tempPath, "missing-file-bundle");
CreateValidBundle(bundlePath);
// Delete a file
File.Delete(Path.Combine(bundlePath, "tokenizer.json"));
var manager = new FileSystemModelBundleManager(_tempPath);
// Act
var result = await manager.VerifyBundleAsync(bundlePath);
// Assert
Assert.False(result.Valid);
Assert.NotEmpty(result.FailedFiles);
Assert.Contains(result.FailedFiles, f => f.Contains("missing"));
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task BundleManager_ListBundles_ReturnsAvailableBundles()
{
// Arrange
CreateValidBundle(Path.Combine(_tempPath, "bundle1"));
CreateValidBundle(Path.Combine(_tempPath, "bundle2"));
var manager = new FileSystemModelBundleManager(_tempPath);
// Act
var bundles = await manager.ListBundlesAsync();
// Assert
Assert.Equal(2, bundles.Count);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task BundleManager_GetManifest_ExistingBundle_ReturnsManifest()
{
// Arrange
CreateValidBundle(Path.Combine(_tempPath, "test-bundle"));
var manager = new FileSystemModelBundleManager(_tempPath);
// Act
var manifest = await manager.GetManifestAsync("test-bundle");
// Assert
Assert.NotNull(manifest);
Assert.Equal("test-model", manifest.Name);
Assert.Equal("Apache-2.0", manifest.License);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task BundleManager_GetManifest_NonExistentBundle_ReturnsNull()
{
// Arrange
var manager = new FileSystemModelBundleManager(_tempPath);
// Act
var manifest = await manager.GetManifestAsync("nonexistent");
// Assert
Assert.Null(manifest);
}
#endregion
#region Offline Replay Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task OfflineReplay_SameInputs_ProducesSameOutput()
{
// Arrange
var request = new LlmCompletionRequest
{
UserPrompt = "Analyze vulnerability impact",
SystemPrompt = "You are a security expert.",
Temperature = 0,
Seed = 42,
MaxTokens = 1024
};
// Simulate first run
var originalResult = await _stubProvider.CompleteAsync(request);
await _cache.SetAsync(request, "stub", originalResult);
// Simulate replay (offline)
var replayResult = await _cache.TryGetAsync(request, "stub");
// Assert
Assert.NotNull(replayResult);
Assert.Equal(originalResult.Content, replayResult.Content);
Assert.True(originalResult.Deterministic);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task OfflineReplay_DifferentInputs_DifferentOutput()
{
// Arrange
var request1 = new LlmCompletionRequest
{
UserPrompt = "Input A",
Temperature = 0,
Seed = 42
};
var request2 = new LlmCompletionRequest
{
UserPrompt = "Input B",
Temperature = 0,
Seed = 42
};
// Act
var result1 = await _stubProvider.CompleteAsync(request1);
var result2 = await _stubProvider.CompleteAsync(request2);
// Assert
Assert.NotEqual(result1.Content, result2.Content);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task Cache_Invalidation_RemovesEntries()
{
// Arrange
var request = new LlmCompletionRequest
{
UserPrompt = "To be invalidated",
Temperature = 0,
Seed = 42
};
var result = new LlmCompletionResult
{
Content = "Cached content",
ModelId = "test-model",
ProviderId = "stub",
Deterministic = true
};
await _cache.SetAsync(request, "stub", result);
// Verify it's cached
var cached = await _cache.TryGetAsync(request, "stub");
Assert.NotNull(cached);
// Act - Invalidate
await _cache.InvalidateAsync("stub");
// Assert
var afterInvalidation = await _cache.TryGetAsync(request, "stub");
Assert.Null(afterInvalidation);
}
#endregion
#region LocalLlmConfig Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public void LocalLlmConfig_DefaultValues_AreCorrect()
{
// Arrange & Act
var config = new LocalLlmConfig
{
ModelPath = "/models/test.gguf",
WeightsDigest = "abc123"
};
// Assert
Assert.Equal(ModelQuantization.Q4_K_M, config.Quantization);
Assert.Equal(4096, config.ContextLength);
Assert.Equal(InferenceDevice.Auto, config.Device);
Assert.Equal(0, config.Temperature);
Assert.Equal(42, config.Seed);
Assert.True(config.FlashAttention);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public void LocalLlmConfig_CustomValues_AreApplied()
{
// Arrange & Act
var config = new LocalLlmConfig
{
ModelPath = "/models/llama3-8b.gguf",
WeightsDigest = "sha256:abc123def456",
Quantization = ModelQuantization.FP16,
ContextLength = 8192,
Device = InferenceDevice.CUDA,
GpuLayers = 32,
Threads = 8,
Temperature = 0,
Seed = 12345,
FlashAttention = false,
MaxTokens = 4096
};
// Assert
Assert.Equal("/models/llama3-8b.gguf", config.ModelPath);
Assert.Equal(ModelQuantization.FP16, config.Quantization);
Assert.Equal(8192, config.ContextLength);
Assert.Equal(InferenceDevice.CUDA, config.Device);
Assert.Equal(32, config.GpuLayers);
Assert.Equal(12345, config.Seed);
}
#endregion
#region Fallback Provider Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task FallbackLlmProvider_FirstAvailable_UsesFirstProvider()
{
// Arrange
var factory = new StubLlmProviderFactory(new Dictionary<string, ILlmProvider>
{
["primary"] = new StubLlmProvider { IsAvailableResult = true, ProviderIdOverride = "primary" },
["fallback"] = new StubLlmProvider { IsAvailableResult = true, ProviderIdOverride = "fallback" }
});
var fallbackProvider = new FallbackLlmProvider(
factory,
new[] { "primary", "fallback" },
NullLogger<FallbackLlmProvider>.Instance);
var request = new LlmCompletionRequest { UserPrompt = "Test" };
// Act
var result = await fallbackProvider.CompleteAsync(request);
// Assert
Assert.Equal("primary", result.ProviderId);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task FallbackLlmProvider_FirstUnavailable_UsesFallback()
{
// Arrange
var factory = new StubLlmProviderFactory(new Dictionary<string, ILlmProvider>
{
["primary"] = new StubLlmProvider { IsAvailableResult = false, ProviderIdOverride = "primary" },
["fallback"] = new StubLlmProvider { IsAvailableResult = true, ProviderIdOverride = "fallback" }
});
var fallbackProvider = new FallbackLlmProvider(
factory,
new[] { "primary", "fallback" },
NullLogger<FallbackLlmProvider>.Instance);
var request = new LlmCompletionRequest { UserPrompt = "Test" };
// Act
var result = await fallbackProvider.CompleteAsync(request);
// Assert
Assert.Equal("fallback", result.ProviderId);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task FallbackLlmProvider_AllUnavailable_ThrowsException()
{
// Arrange
var factory = new StubLlmProviderFactory(new Dictionary<string, ILlmProvider>
{
["primary"] = new StubLlmProvider { IsAvailableResult = false },
["fallback"] = new StubLlmProvider { IsAvailableResult = false }
});
var fallbackProvider = new FallbackLlmProvider(
factory,
new[] { "primary", "fallback" },
NullLogger<FallbackLlmProvider>.Instance);
var request = new LlmCompletionRequest { UserPrompt = "Test" };
// Act & Assert
await Assert.ThrowsAsync<InvalidOperationException>(() =>
fallbackProvider.CompleteAsync(request));
}
#endregion
#region Helper Methods
private void CreateValidBundle(string bundlePath)
{
Directory.CreateDirectory(bundlePath);
// Create model file
var modelContent = "fake model weights for testing";
var modelPath = Path.Combine(bundlePath, "model.gguf");
File.WriteAllText(modelPath, modelContent);
// Create tokenizer file
var tokenizerContent = "{\"vocab_size\": 32000}";
var tokenizerPath = Path.Combine(bundlePath, "tokenizer.json");
File.WriteAllText(tokenizerPath, tokenizerContent);
// Compute digests
using var sha256 = SHA256.Create();
var modelDigest = Convert.ToHexStringLower(sha256.ComputeHash(Encoding.UTF8.GetBytes(modelContent)));
var tokenizerDigest = Convert.ToHexStringLower(sha256.ComputeHash(Encoding.UTF8.GetBytes(tokenizerContent)));
// Create manifest
var manifest = new ModelBundleManifest
{
Name = "test-model",
License = "Apache-2.0",
SizeCategory = "7B",
Quantizations = new[] { "Q4_K_M", "FP16" },
CreatedAt = DateTime.UtcNow.ToString("o"),
Files = new[]
{
new BundleFile { Path = "model.gguf", Digest = modelDigest, Size = modelContent.Length, Type = "weights" },
new BundleFile { Path = "tokenizer.json", Digest = tokenizerDigest, Size = tokenizerContent.Length, Type = "tokenizer" }
}
};
var manifestJson = JsonSerializer.Serialize(manifest, new JsonSerializerOptions { WriteIndented = true });
File.WriteAllText(Path.Combine(bundlePath, "manifest.json"), manifestJson);
}
#endregion
#region Stub Implementations
private sealed class StubLlmProvider : ILlmProvider
{
public string ProviderId => ProviderIdOverride ?? "stub";
public string? ProviderIdOverride { get; set; }
public bool IsAvailableResult { get; set; } = true;
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
=> Task.FromResult(IsAvailableResult);
public Task<LlmCompletionResult> CompleteAsync(
LlmCompletionRequest request,
CancellationToken cancellationToken = default)
{
// Generate deterministic output based on input hash
using var sha = SHA256.Create();
var inputHash = Convert.ToHexStringLower(
sha.ComputeHash(Encoding.UTF8.GetBytes(
$"{request.SystemPrompt}||{request.UserPrompt}||{request.Seed}")));
var content = $"Deterministic response for input hash: {inputHash[..16]}";
return Task.FromResult(new LlmCompletionResult
{
Content = content,
ModelId = "stub-model",
ProviderId = ProviderId,
Deterministic = request.Temperature == 0,
InputTokens = request.UserPrompt.Length / 4,
OutputTokens = content.Length / 4,
FinishReason = "stop",
RequestId = request.RequestId
});
}
public async IAsyncEnumerable<LlmStreamChunk> CompleteStreamAsync(
LlmCompletionRequest request,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var words = new[] { "This ", "is ", "a ", "streaming ", "response." };
foreach (var word in words)
{
await Task.Delay(10, cancellationToken);
yield return new LlmStreamChunk { Content = word, IsFinal = false };
}
yield return new LlmStreamChunk { Content = "", IsFinal = true, FinishReason = "stop" };
}
public void Dispose() { }
}
private sealed class CallCountingLlmProvider : ILlmProvider
{
public string ProviderId => "counting";
public int CallCount { get; private set; }
public Task<bool> IsAvailableAsync(CancellationToken cancellationToken = default)
=> Task.FromResult(true);
public Task<LlmCompletionResult> CompleteAsync(
LlmCompletionRequest request,
CancellationToken cancellationToken = default)
{
CallCount++;
return Task.FromResult(new LlmCompletionResult
{
Content = $"Response #{CallCount}",
ModelId = "counting-model",
ProviderId = ProviderId,
Deterministic = request.Temperature == 0,
OutputTokens = 5
});
}
public async IAsyncEnumerable<LlmStreamChunk> CompleteStreamAsync(
LlmCompletionRequest request,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
CallCount++;
yield return new LlmStreamChunk { Content = "Response", IsFinal = true };
await Task.CompletedTask;
}
public void Dispose() { }
}
private sealed class StubLlmProviderFactory : ILlmProviderFactory
{
private readonly Dictionary<string, ILlmProvider> _providers;
public StubLlmProviderFactory(Dictionary<string, ILlmProvider> providers)
{
_providers = providers;
}
public IReadOnlyList<string> AvailableProviders => _providers.Keys.ToList();
public ILlmProvider GetProvider(string providerId)
{
if (_providers.TryGetValue(providerId, out var provider))
return provider;
throw new InvalidOperationException($"Provider '{providerId}' not found");
}
public ILlmProvider GetDefaultProvider() => _providers.Values.First();
}
#endregion
}

View File

@@ -0,0 +1,834 @@
using FluentAssertions;
using StellaOps.AdvisoryAI.PolicyStudio;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
/// <summary>
/// Integration tests for Policy Studio NL→rule→test round-trip and conflict detection.
/// Sprint: SPRINT_20251226_017_AI_policy_copilot
/// Task: POLICY-25
/// </summary>
public sealed class PolicyStudioIntegrationTests
{
#region NL Intent Rule Round-Trip Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ParseAndGenerate_OverrideRule_ProducesValidLatticeRule()
{
// Arrange
var parser = new StubPolicyIntentParser();
var generator = new StubPolicyRuleGenerator();
var synthesizer = new StubTestCaseSynthesizer();
var naturalLanguage = "Block all critical vulnerabilities that are reachable";
// Act - Parse NL to intent
var parseResult = await parser.ParseAsync(naturalLanguage);
parseResult.Success.Should().BeTrue();
parseResult.Intent.IntentType.Should().Be(PolicyIntentType.OverrideRule);
// Act - Generate rules from intent
var ruleResult = await generator.GenerateAsync(parseResult.Intent);
ruleResult.Success.Should().BeTrue();
ruleResult.Rules.Should().NotBeEmpty();
// Assert - Rules have correct structure
var rule = ruleResult.Rules[0];
rule.LatticeExpression.Should().Contain("REACHABLE");
rule.Disposition.Should().Be("block");
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ParseAndGenerate_ExceptionRule_ProducesValidLatticeRule()
{
// Arrange
var parser = new StubPolicyIntentParser();
var generator = new StubPolicyRuleGenerator();
var naturalLanguage = "Allow vulnerabilities with vendor VEX not_affected status";
// Act
var parseResult = await parser.ParseAsync(naturalLanguage, new PolicyParseContext
{
DefaultScope = "all"
});
var ruleResult = await generator.GenerateAsync(parseResult.Intent);
// Assert
ruleResult.Success.Should().BeTrue();
var rule = ruleResult.Rules[0];
rule.Disposition.Should().Be("allow");
rule.Conditions.Should().Contain(c => c.Field == "vex_status");
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task FullRoundTrip_NLToRuleToTest_ProducesValidTestCases()
{
// Arrange
var parser = new StubPolicyIntentParser();
var generator = new StubPolicyRuleGenerator();
var synthesizer = new StubTestCaseSynthesizer();
var naturalLanguage = "Block critical reachable vulnerabilities without VEX";
// Act - Full round-trip
var parseResult = await parser.ParseAsync(naturalLanguage);
var ruleResult = await generator.GenerateAsync(parseResult.Intent);
var testCases = await synthesizer.SynthesizeAsync(ruleResult.Rules);
// Assert
testCases.Should().NotBeEmpty();
testCases.Should().Contain(t => t.Type == TestCaseType.Positive);
testCases.Should().Contain(t => t.Type == TestCaseType.Negative);
}
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("Block all high severity findings", PolicyIntentType.OverrideRule)]
[InlineData("Escalate critical vulnerabilities to security team", PolicyIntentType.EscalationRule)]
[InlineData("Allow exceptions for internal-only services", PolicyIntentType.ExceptionCondition)]
[InlineData("Set severity threshold to 7.0 for blocking", PolicyIntentType.ThresholdRule)]
public async Task ParseAsync_RecognizesIntentTypes(string input, PolicyIntentType expectedType)
{
// Arrange
var parser = new StubPolicyIntentParser(expectedType);
// Act
var result = await parser.ParseAsync(input);
// Assert
result.Success.Should().BeTrue();
result.Intent.IntentType.Should().Be(expectedType);
}
#endregion
#region Conflict Detection Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ValidateAsync_DetectsConflictingRules()
{
// Arrange
var generator = new StubPolicyRuleGenerator();
var conflictingRules = new List<LatticeRule>
{
CreateRule("rule-1", "REACHABLE ∧ PRESENT", "block", priority: 10),
CreateRule("rule-2", "REACHABLE ∧ PRESENT", "allow", priority: 20)
};
// Act
var validationResult = await generator.ValidateAsync(conflictingRules);
// Assert
validationResult.Valid.Should().BeFalse();
validationResult.Conflicts.Should().NotBeEmpty();
validationResult.Conflicts[0].RuleId1.Should().Be("rule-1");
validationResult.Conflicts[0].RuleId2.Should().Be("rule-2");
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ValidateAsync_NoConflict_WhenDifferentConditions()
{
// Arrange
var generator = new StubPolicyRuleGenerator();
var nonConflictingRules = new List<LatticeRule>
{
CreateRule("rule-1", "REACHABLE ∧ PRESENT", "block", priority: 10),
CreateRule("rule-2", "¬REACHABLE ∧ PRESENT", "allow", priority: 20)
};
// Act
var validationResult = await generator.ValidateAsync(nonConflictingRules);
// Assert
validationResult.Conflicts.Should().BeEmpty();
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ValidateAsync_DetectsUnreachableConditions()
{
// Arrange
var generator = new StubPolicyRuleGenerator();
var rules = new List<LatticeRule>
{
CreateRule("rule-1", "REACHABLE ∧ ¬REACHABLE", "block", priority: 10) // Contradiction
};
// Act
var validationResult = await generator.ValidateAsync(rules);
// Assert
validationResult.UnreachableConditions.Should().NotBeEmpty();
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ValidateAsync_ReportsCoverageMetric()
{
// Arrange
var generator = new StubPolicyRuleGenerator();
var rules = new List<LatticeRule>
{
CreateRule("rule-1", "REACHABLE", "block", priority: 10),
CreateRule("rule-2", "PRESENT", "warn", priority: 20),
CreateRule("rule-3", "FIXED", "allow", priority: 30)
};
// Act
var validationResult = await generator.ValidateAsync(rules);
// Assert
validationResult.Coverage.Should().BeGreaterThan(0);
validationResult.Coverage.Should().BeLessThanOrEqualTo(1.0);
}
#endregion
#region Test Case Synthesis Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SynthesizeAsync_GeneratesPositiveTests()
{
// Arrange
var synthesizer = new StubTestCaseSynthesizer();
var rules = new List<LatticeRule>
{
CreateRule("rule-1", "REACHABLE ∧ PRESENT", "block", priority: 10)
};
// Act
var testCases = await synthesizer.SynthesizeAsync(rules);
// Assert
var positiveTests = testCases.Where(t => t.Type == TestCaseType.Positive).ToList();
positiveTests.Should().NotBeEmpty();
positiveTests.Should().AllSatisfy(t =>
{
t.ExpectedDisposition.Should().Be("block");
t.TargetRuleIds.Should().Contain("rule-1");
});
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SynthesizeAsync_GeneratesNegativeTests()
{
// Arrange
var synthesizer = new StubTestCaseSynthesizer();
var rules = new List<LatticeRule>
{
CreateRule("rule-1", "REACHABLE ∧ PRESENT", "block", priority: 10)
};
// Act
var testCases = await synthesizer.SynthesizeAsync(rules);
// Assert
var negativeTests = testCases.Where(t => t.Type == TestCaseType.Negative).ToList();
negativeTests.Should().NotBeEmpty();
negativeTests.Should().AllSatisfy(t =>
{
t.Description.Should().NotBeNullOrEmpty();
});
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SynthesizeAsync_GeneratesBoundaryTests()
{
// Arrange
var synthesizer = new StubTestCaseSynthesizer();
var rules = new List<LatticeRule>
{
CreateRule("rule-1", "cvss_score >= 7.0", "block", priority: 10)
};
// Act
var testCases = await synthesizer.SynthesizeAsync(rules);
// Assert
var boundaryTests = testCases.Where(t => t.Type == TestCaseType.Boundary).ToList();
boundaryTests.Should().NotBeEmpty();
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task SynthesizeAsync_GeneratesConflictTests_ForOverlappingRules()
{
// Arrange
var synthesizer = new StubTestCaseSynthesizer();
var rules = new List<LatticeRule>
{
CreateRule("rule-1", "REACHABLE", "block", priority: 10),
CreateRule("rule-2", "REACHABLE ∧ HAS_VEX", "allow", priority: 20)
};
// Act
var testCases = await synthesizer.SynthesizeAsync(rules);
// Assert
var conflictTests = testCases.Where(t => t.Type == TestCaseType.Conflict).ToList();
conflictTests.Should().NotBeEmpty();
conflictTests.Should().AllSatisfy(t =>
{
t.TargetRuleIds.Count.Should().BeGreaterThan(1);
});
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RunTestsAsync_PassesWithMatchingRules()
{
// Arrange
var synthesizer = new StubTestCaseSynthesizer();
var rules = new List<LatticeRule>
{
CreateRule("rule-1", "REACHABLE", "block", priority: 10)
};
var testCases = await synthesizer.SynthesizeAsync(rules);
// Act
var result = await synthesizer.RunTestsAsync(testCases, rules);
// Assert
result.Success.Should().BeTrue();
result.Passed.Should().Be(result.Total);
result.Failed.Should().Be(0);
}
#endregion
#region Edge Cases
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ParseAsync_WithAmbiguousInput_ReturnsAlternatives()
{
// Arrange
var parser = new StubPolicyIntentParser(ambiguous: true);
var ambiguousInput = "Block vulnerabilities in production";
// Act
var result = await parser.ParseAsync(ambiguousInput);
// Assert
result.Intent.Alternatives.Should().NotBeNullOrEmpty();
result.Intent.ClarifyingQuestions.Should().NotBeNullOrEmpty();
result.Intent.Confidence.Should().BeLessThan(0.9);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GenerateAsync_WithEmptyConditions_ReturnsError()
{
// Arrange
var generator = new StubPolicyRuleGenerator();
var intent = new PolicyIntent
{
IntentId = "empty-intent",
IntentType = PolicyIntentType.OverrideRule,
OriginalInput = "Block everything",
Conditions = [],
Actions = [],
Scope = "all",
Priority = 100,
Confidence = 0.5
};
// Act
var result = await generator.GenerateAsync(intent);
// Assert
result.Success.Should().BeFalse();
result.Errors.Should().NotBeNullOrEmpty();
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ClarifyAsync_UpdatesIntentWithClarification()
{
// Arrange
var parser = new StubPolicyIntentParser(ambiguous: true);
var initialResult = await parser.ParseAsync("Block vulnerabilities");
// Act
var clarifiedResult = await parser.ClarifyAsync(
initialResult.Intent.IntentId,
"Only critical severity");
// Assert
clarifiedResult.Intent.Confidence.Should().BeGreaterThan(initialResult.Intent.Confidence);
clarifiedResult.Intent.ClarifyingQuestions.Should().BeNullOrEmpty();
}
#endregion
#region Helper Methods
private static LatticeRule CreateRule(string ruleId, string expression, string disposition, int priority)
=> new()
{
RuleId = ruleId,
Name = $"Test Rule {ruleId}",
Description = $"Test rule with expression: {expression}",
LatticeExpression = expression,
Conditions = ParseConditions(expression),
Disposition = disposition,
Priority = priority,
Scope = "all",
Enabled = true
};
private static IReadOnlyList<PolicyCondition> ParseConditions(string expression)
{
var conditions = new List<PolicyCondition>();
if (expression.Contains("REACHABLE", StringComparison.OrdinalIgnoreCase))
{
conditions.Add(new PolicyCondition
{
Field = "reachable",
Operator = "equals",
Value = true,
Connector = expression.Contains("∧") ? "and" : null
});
}
if (expression.Contains("PRESENT", StringComparison.OrdinalIgnoreCase))
{
conditions.Add(new PolicyCondition
{
Field = "present",
Operator = "equals",
Value = true
});
}
if (expression.Contains("cvss_score", StringComparison.OrdinalIgnoreCase))
{
conditions.Add(new PolicyCondition
{
Field = "cvss_score",
Operator = "greater_than_or_equal",
Value = 7.0
});
}
return conditions;
}
#endregion
#region Stub Implementations
private sealed class StubPolicyIntentParser : IPolicyIntentParser
{
private readonly PolicyIntentType _defaultType;
private readonly bool _ambiguous;
private readonly Dictionary<string, PolicyIntent> _intents = new();
public StubPolicyIntentParser(
PolicyIntentType defaultType = PolicyIntentType.OverrideRule,
bool ambiguous = false)
{
_defaultType = defaultType;
_ambiguous = ambiguous;
}
public Task<PolicyParseResult> ParseAsync(
string naturalLanguageInput,
PolicyParseContext? context = null,
CancellationToken cancellationToken = default)
{
var intentId = $"intent-{Guid.NewGuid():N}";
var confidence = _ambiguous ? 0.7 : 0.95;
var conditions = new List<PolicyCondition>();
if (naturalLanguageInput.Contains("critical", StringComparison.OrdinalIgnoreCase))
{
conditions.Add(new PolicyCondition
{
Field = "severity",
Operator = "equals",
Value = "critical"
});
}
if (naturalLanguageInput.Contains("reachable", StringComparison.OrdinalIgnoreCase))
{
conditions.Add(new PolicyCondition
{
Field = "reachable",
Operator = "equals",
Value = true
});
}
if (naturalLanguageInput.Contains("VEX", StringComparison.OrdinalIgnoreCase))
{
conditions.Add(new PolicyCondition
{
Field = "vex_status",
Operator = "equals",
Value = "not_affected"
});
}
var intent = new PolicyIntent
{
IntentId = intentId,
IntentType = _defaultType,
OriginalInput = naturalLanguageInput,
Conditions = conditions,
Actions = [new PolicyAction
{
ActionType = "set_verdict",
Parameters = new Dictionary<string, object> { ["verdict"] = "block" }
}],
Scope = context?.DefaultScope ?? "all",
Priority = 100,
Confidence = confidence,
Alternatives = _ambiguous ? [CreateAlternativeIntent(naturalLanguageInput)] : null,
ClarifyingQuestions = _ambiguous ? ["What severity levels should be affected?"] : null
};
_intents[intentId] = intent;
return Task.FromResult(new PolicyParseResult
{
Intent = intent,
Success = true,
ModelId = "stub-parser-v1",
ParsedAt = DateTime.UtcNow.ToString("O")
});
}
public Task<PolicyParseResult> ClarifyAsync(
string intentId,
string clarification,
CancellationToken cancellationToken = default)
{
var original = _intents.GetValueOrDefault(intentId);
if (original is null)
{
throw new InvalidOperationException($"Intent {intentId} not found");
}
var clarified = original with
{
Confidence = 0.95,
ClarifyingQuestions = null,
Alternatives = null
};
return Task.FromResult(new PolicyParseResult
{
Intent = clarified,
Success = true,
ModelId = "stub-parser-v1",
ParsedAt = DateTime.UtcNow.ToString("O")
});
}
private PolicyIntent CreateAlternativeIntent(string input) => new()
{
IntentId = $"alt-{Guid.NewGuid():N}",
IntentType = PolicyIntentType.ExceptionCondition,
OriginalInput = input,
Conditions = [],
Actions = [],
Scope = "all",
Priority = 50,
Confidence = 0.5
};
}
private sealed class StubPolicyRuleGenerator : IPolicyRuleGenerator
{
public Task<RuleGenerationResult> GenerateAsync(
PolicyIntent intent,
CancellationToken cancellationToken = default)
{
if (intent.Conditions.Count == 0)
{
return Task.FromResult(new RuleGenerationResult
{
Rules = [],
Success = false,
Warnings = [],
Errors = ["Intent must have at least one condition"],
IntentId = intent.IntentId,
GeneratedAt = DateTime.UtcNow.ToString("O")
});
}
var expression = BuildLatticeExpression(intent.Conditions);
var disposition = intent.Actions.FirstOrDefault()?.Parameters.GetValueOrDefault("verdict")?.ToString() ?? "warn";
var rule = new LatticeRule
{
RuleId = $"rule-{Guid.NewGuid():N}",
Name = $"Generated from: {intent.OriginalInput[..Math.Min(30, intent.OriginalInput.Length)]}",
Description = intent.OriginalInput,
LatticeExpression = expression,
Conditions = intent.Conditions,
Disposition = disposition,
Priority = intent.Priority,
Scope = intent.Scope,
Enabled = true
};
return Task.FromResult(new RuleGenerationResult
{
Rules = [rule],
Success = true,
Warnings = [],
IntentId = intent.IntentId,
GeneratedAt = DateTime.UtcNow.ToString("O")
});
}
public Task<RuleValidationResult> ValidateAsync(
IReadOnlyList<LatticeRule> rules,
IReadOnlyList<string>? existingRuleIds = null,
CancellationToken cancellationToken = default)
{
var conflicts = new List<RuleConflict>();
var unreachable = new List<string>();
// Check for conflicts
for (int i = 0; i < rules.Count; i++)
{
for (int j = i + 1; j < rules.Count; j++)
{
if (HasConflict(rules[i], rules[j]))
{
conflicts.Add(new RuleConflict
{
RuleId1 = rules[i].RuleId,
RuleId2 = rules[j].RuleId,
Description = "Rules have overlapping conditions with different dispositions",
SuggestedResolution = "Adjust priority or narrow conditions",
Severity = "error"
});
}
}
// Check for unreachable conditions (contradictions)
if (rules[i].LatticeExpression.Contains("∧ ¬") &&
rules[i].LatticeExpression.Split("∧").Any(p =>
p.Trim().StartsWith("¬") && rules[i].LatticeExpression.Contains(p.Trim()[1..])))
{
unreachable.Add($"Rule {rules[i].RuleId} has contradictory conditions");
}
}
var coverage = Math.Min(1.0, rules.Count * 0.2);
return Task.FromResult(new RuleValidationResult
{
Valid = conflicts.Count == 0 && unreachable.Count == 0,
Conflicts = conflicts,
UnreachableConditions = unreachable,
PotentialLoops = [],
Coverage = coverage
});
}
private static bool HasConflict(LatticeRule rule1, LatticeRule rule2)
{
// Simplified conflict detection
var sameConditions = rule1.LatticeExpression == rule2.LatticeExpression;
var differentDispositions = rule1.Disposition != rule2.Disposition;
return sameConditions && differentDispositions;
}
private static string BuildLatticeExpression(IReadOnlyList<PolicyCondition> conditions)
{
var parts = conditions.Select(c =>
{
var atom = c.Field.ToUpperInvariant() switch
{
"REACHABLE" => "REACHABLE",
"PRESENT" => "PRESENT",
"SEVERITY" => c.Value?.ToString()?.ToUpperInvariant() ?? "CRITICAL",
"VEX_STATUS" => "HAS_VEX",
_ => c.Field.ToUpperInvariant()
};
return c.Operator == "not_equals" || c.Value?.Equals(false) == true
? $"¬{atom}"
: atom;
});
return string.Join(" ∧ ", parts);
}
}
private sealed class StubTestCaseSynthesizer : ITestCaseSynthesizer
{
public Task<IReadOnlyList<PolicyTestCase>> SynthesizeAsync(
IReadOnlyList<LatticeRule> rules,
CancellationToken cancellationToken = default)
{
var testCases = new List<PolicyTestCase>();
var testId = 0;
foreach (var rule in rules)
{
// Positive test
testCases.Add(new PolicyTestCase
{
TestCaseId = $"test-{++testId}",
Name = $"Positive test for {rule.Name}",
Type = TestCaseType.Positive,
Input = BuildPositiveInput(rule),
ExpectedDisposition = rule.Disposition,
TargetRuleIds = [rule.RuleId],
Description = $"Verifies rule matches when conditions are met"
});
// Negative test
testCases.Add(new PolicyTestCase
{
TestCaseId = $"test-{++testId}",
Name = $"Negative test for {rule.Name}",
Type = TestCaseType.Negative,
Input = BuildNegativeInput(rule),
ExpectedDisposition = "no_match",
TargetRuleIds = [rule.RuleId],
Description = $"Verifies rule does not match when conditions are not met"
});
// Boundary test for numeric conditions
if (rule.LatticeExpression.Contains("cvss_score") || rule.LatticeExpression.Contains(">="))
{
testCases.Add(new PolicyTestCase
{
TestCaseId = $"test-{++testId}",
Name = $"Boundary test for {rule.Name}",
Type = TestCaseType.Boundary,
Input = BuildBoundaryInput(rule),
ExpectedDisposition = rule.Disposition,
TargetRuleIds = [rule.RuleId],
Description = $"Verifies rule at boundary values"
});
}
}
// Conflict tests for overlapping rules
for (int i = 0; i < rules.Count; i++)
{
for (int j = i + 1; j < rules.Count; j++)
{
if (RulesOverlap(rules[i], rules[j]))
{
testCases.Add(new PolicyTestCase
{
TestCaseId = $"test-{++testId}",
Name = $"Conflict test: {rules[i].Name} vs {rules[j].Name}",
Type = TestCaseType.Conflict,
Input = BuildOverlapInput(rules[i], rules[j]),
ExpectedDisposition = rules[i].Priority > rules[j].Priority
? rules[i].Disposition
: rules[j].Disposition,
TargetRuleIds = [rules[i].RuleId, rules[j].RuleId],
Description = $"Verifies priority resolution when both rules match"
});
}
}
}
return Task.FromResult<IReadOnlyList<PolicyTestCase>>(testCases);
}
public Task<TestRunResult> RunTestsAsync(
IReadOnlyList<PolicyTestCase> testCases,
IReadOnlyList<LatticeRule> rules,
CancellationToken cancellationToken = default)
{
var results = new List<TestCaseResult>();
foreach (var testCase in testCases)
{
var passed = true; // Simplified - stub always passes
results.Add(new TestCaseResult
{
TestCaseId = testCase.TestCaseId,
Passed = passed,
Expected = testCase.ExpectedDisposition,
Actual = testCase.ExpectedDisposition
});
}
return Task.FromResult(new TestRunResult
{
Total = testCases.Count,
Passed = results.Count(r => r.Passed),
Failed = results.Count(r => !r.Passed),
Results = results,
RunAt = DateTime.UtcNow.ToString("O")
});
}
private static IReadOnlyDictionary<string, object> BuildPositiveInput(LatticeRule rule)
{
var input = new Dictionary<string, object>();
if (rule.LatticeExpression.Contains("REACHABLE")) input["reachable"] = true;
if (rule.LatticeExpression.Contains("PRESENT")) input["present"] = true;
if (rule.LatticeExpression.Contains("HAS_VEX")) input["has_vex"] = true;
return input;
}
private static IReadOnlyDictionary<string, object> BuildNegativeInput(LatticeRule rule)
{
var input = new Dictionary<string, object>();
if (rule.LatticeExpression.Contains("REACHABLE")) input["reachable"] = false;
if (rule.LatticeExpression.Contains("PRESENT")) input["present"] = false;
return input;
}
private static IReadOnlyDictionary<string, object> BuildBoundaryInput(LatticeRule rule)
{
return new Dictionary<string, object>
{
["cvss_score"] = 7.0
};
}
private static IReadOnlyDictionary<string, object> BuildOverlapInput(LatticeRule rule1, LatticeRule rule2)
{
var input = new Dictionary<string, object>();
input["reachable"] = true;
input["present"] = true;
input["has_vex"] = true;
return input;
}
private static bool RulesOverlap(LatticeRule rule1, LatticeRule rule2)
{
// Simplified overlap detection
return rule1.LatticeExpression.Contains("REACHABLE") &&
rule2.LatticeExpression.Contains("REACHABLE");
}
}
#endregion
}

View File

@@ -0,0 +1,791 @@
using StellaOps.AdvisoryAI.Remediation;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
/// <summary>
/// Integration tests for remediation plan generation and PR creation.
/// Sprint: SPRINT_20251226_016_AI_remedy_autopilot
/// Task: REMEDY-25
/// </summary>
public sealed class RemediationIntegrationTests
{
#region Plan Generation Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_ValidRequest_ReturnsPlan()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.NotNull(plan);
Assert.Equal(request.FindingId, plan.Request.FindingId);
Assert.NotEmpty(plan.Steps);
Assert.NotEmpty(plan.PlanId);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_BumpRemediation_GeneratesBumpSteps()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest() with
{
RemediationType = RemediationType.Bump,
ComponentPurl = "pkg:npm/lodash@4.17.20"
};
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Contains(plan.Steps, s => s.ActionType == "update_package");
Assert.True(plan.ExpectedDelta.Upgraded.Count > 0);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_UpgradeRemediation_GeneratesUpgradeSteps()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest() with
{
RemediationType = RemediationType.Upgrade,
ComponentPurl = "pkg:oci/alpine@3.18"
};
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Contains(plan.Steps, s => s.ActionType == "update_base_image");
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_ConfigRemediation_GeneratesConfigSteps()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest() with
{
RemediationType = RemediationType.Config,
VulnerabilityId = "CVE-2021-44228" // Log4Shell
};
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Contains(plan.Steps, s => s.ActionType == "update_config");
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_AssessesRiskCorrectly_PatchVersion()
{
// Arrange
var planner = new StubRemediationPlanner(patchVersionBump: true);
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Equal(RemediationRisk.Low, plan.RiskAssessment);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_AssessesRiskCorrectly_MajorVersion()
{
// Arrange
var planner = new StubRemediationPlanner(majorVersionBump: true);
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Equal(RemediationRisk.High, plan.RiskAssessment);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_IncludesExpectedSbomDelta()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.NotNull(plan.ExpectedDelta);
Assert.True(plan.ExpectedDelta.NetVulnerabilityChange < 0); // Should improve
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_IncludesTestRequirements()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.NotNull(plan.TestRequirements);
Assert.NotEmpty(plan.TestRequirements.TestSuites);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_IncludesInputHashes()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.NotEmpty(plan.InputHashes);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ValidatePlanAsync_ExistingPlan_ReturnsTrue()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest();
var plan = await planner.GeneratePlanAsync(request);
// Act
var isValid = await planner.ValidatePlanAsync(plan.PlanId);
// Assert
Assert.True(isValid);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ValidatePlanAsync_NonexistentPlan_ReturnsFalse()
{
// Arrange
var planner = new StubRemediationPlanner();
// Act
var isValid = await planner.ValidatePlanAsync("nonexistent-plan");
// Assert
Assert.False(isValid);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GetPlanAsync_ExistingPlan_ReturnsPlan()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest();
var plan = await planner.GeneratePlanAsync(request);
// Act
var retrieved = await planner.GetPlanAsync(plan.PlanId);
// Assert
Assert.NotNull(retrieved);
Assert.Equal(plan.PlanId, retrieved.PlanId);
}
#endregion
#region PR Generation Tests (Mocked SCM)
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePullRequestAsync_ValidPlan_CreatesPR()
{
// Arrange
var prGenerator = new StubPullRequestGenerator();
var plan = CreateTestPlan();
// Act
var result = await prGenerator.CreatePullRequestAsync(plan);
// Assert
Assert.NotNull(result);
Assert.NotEmpty(result.PrId);
Assert.True(result.PrNumber > 0);
Assert.NotEmpty(result.Url);
Assert.NotEmpty(result.BranchName);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePullRequestAsync_SetsBranchNameFromPlan()
{
// Arrange
var prGenerator = new StubPullRequestGenerator();
var plan = CreateTestPlan();
// Act
var result = await prGenerator.CreatePullRequestAsync(plan);
// Assert
Assert.Contains("stellaops-fix", result.BranchName);
Assert.Contains(plan.Request.VulnerabilityId.ToLowerInvariant(), result.BranchName);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task CreatePullRequestAsync_InitialStatus_IsOpen()
{
// Arrange
var prGenerator = new StubPullRequestGenerator();
var plan = CreateTestPlan();
// Act
var result = await prGenerator.CreatePullRequestAsync(plan);
// Assert
Assert.Equal(PullRequestStatus.Open, result.Status);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GetStatusAsync_ExistingPR_ReturnsStatus()
{
// Arrange
var prGenerator = new StubPullRequestGenerator();
var plan = CreateTestPlan();
var pr = await prGenerator.CreatePullRequestAsync(plan);
// Act
var status = await prGenerator.GetStatusAsync(pr.PrId);
// Assert
Assert.NotNull(status);
Assert.Equal(pr.PrId, status.PrId);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task UpdateWithDeltaVerdictAsync_UpdatesPR()
{
// Arrange
var prGenerator = new StubPullRequestGenerator();
var plan = CreateTestPlan();
var pr = await prGenerator.CreatePullRequestAsync(plan);
var deltaVerdict = new DeltaVerdictResult
{
Improved = true,
VulnerabilitiesFixed = 3,
VulnerabilitiesIntroduced = 0,
VerdictId = "delta-001",
ComputedAt = DateTime.UtcNow.ToString("o")
};
// Act
await prGenerator.UpdateWithDeltaVerdictAsync(pr.PrId, deltaVerdict);
var updated = await prGenerator.GetStatusAsync(pr.PrId);
// Assert
Assert.NotNull(updated.DeltaVerdict);
Assert.Equal(3, updated.DeltaVerdict.VulnerabilitiesFixed);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task ClosePullRequestAsync_ClosesPR()
{
// Arrange
var prGenerator = new StubPullRequestGenerator();
var plan = CreateTestPlan();
var pr = await prGenerator.CreatePullRequestAsync(plan);
// Act
await prGenerator.ClosePullRequestAsync(pr.PrId, "Superseded by manual fix");
var status = await prGenerator.GetStatusAsync(pr.PrId);
// Assert
Assert.Equal(PullRequestStatus.Closed, status.Status);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public void ScmType_GitHub_ReturnsCorrectType()
{
// Arrange
var prGenerator = new StubPullRequestGenerator { ScmTypeOverride = "github" };
// Assert
Assert.Equal("github", prGenerator.ScmType);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public void ScmType_GitLab_ReturnsCorrectType()
{
// Arrange
var prGenerator = new StubPullRequestGenerator { ScmTypeOverride = "gitlab" };
// Assert
Assert.Equal("gitlab", prGenerator.ScmType);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public void ScmType_AzureDevOps_ReturnsCorrectType()
{
// Arrange
var prGenerator = new StubPullRequestGenerator { ScmTypeOverride = "azure-devops" };
// Assert
Assert.Equal("azure-devops", prGenerator.ScmType);
}
#endregion
#region Fallback Handling Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_BuildFails_SetsSuggestionAuthority()
{
// Arrange
var planner = new StubRemediationPlanner(buildWillFail: true);
var request = CreateTestRequest() with { AutoCreatePr = true };
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Equal(RemediationAuthority.Suggestion, plan.Authority);
Assert.False(plan.PrReady);
Assert.NotNull(plan.NotReadyReason);
Assert.Contains("build", plan.NotReadyReason.ToLower());
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_TestsFail_SetsSuggestionAuthority()
{
// Arrange
var planner = new StubRemediationPlanner(testsWillFail: true);
var request = CreateTestRequest() with { AutoCreatePr = true };
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Equal(RemediationAuthority.Suggestion, plan.Authority);
Assert.False(plan.PrReady);
Assert.NotNull(plan.NotReadyReason);
Assert.Contains("test", plan.NotReadyReason.ToLower());
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_NoAutoCreatePr_SetsDraftAuthority()
{
// Arrange
var planner = new StubRemediationPlanner();
var request = CreateTestRequest() with { AutoCreatePr = false };
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Equal(RemediationAuthority.Draft, plan.Authority);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_AllVerificationsPassed_SetsVerifiedAuthority()
{
// Arrange
var planner = new StubRemediationPlanner(allVerificationsPassed: true);
var request = CreateTestRequest() with { AutoCreatePr = true };
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.Equal(RemediationAuthority.Verified, plan.Authority);
Assert.True(plan.PrReady);
Assert.Null(plan.NotReadyReason);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_BreakingChanges_ReducesConfidence()
{
// Arrange
var planner = new StubRemediationPlanner(hasBreakingChanges: true);
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.True(plan.ConfidenceScore < 0.8);
}
#endregion
#region Confidence Score Tests
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_PatchVersion_HighConfidence()
{
// Arrange
var planner = new StubRemediationPlanner(patchVersionBump: true);
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.True(plan.ConfidenceScore >= 0.9);
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GeneratePlanAsync_MajorVersion_LowerConfidence()
{
// Arrange
var planner = new StubRemediationPlanner(majorVersionBump: true);
var request = CreateTestRequest();
// Act
var plan = await planner.GeneratePlanAsync(request);
// Assert
Assert.True(plan.ConfidenceScore < 0.7);
}
#endregion
#region Helper Methods
private static RemediationPlanRequest CreateTestRequest()
{
return new RemediationPlanRequest
{
FindingId = "finding-001",
ArtifactDigest = "sha256:abc123",
VulnerabilityId = "CVE-2024-1234",
ComponentPurl = "pkg:npm/lodash@4.17.20",
RemediationType = RemediationType.Auto,
RepositoryUrl = "https://github.com/test/repo",
TargetBranch = "main",
AutoCreatePr = false,
CorrelationId = Guid.NewGuid().ToString()
};
}
private static RemediationPlan CreateTestPlan()
{
return new RemediationPlan
{
PlanId = $"plan-{Guid.NewGuid():N}",
Request = CreateTestRequest(),
Steps = new[]
{
new RemediationStep
{
Order = 1,
ActionType = "update_package",
FilePath = "package.json",
Description = "Update lodash from 4.17.20 to 4.17.21",
PreviousValue = "4.17.20",
NewValue = "4.17.21",
Risk = RemediationRisk.Low
}
},
ExpectedDelta = new ExpectedSbomDelta
{
Added = Array.Empty<string>(),
Removed = Array.Empty<string>(),
Upgraded = new Dictionary<string, string>
{
["pkg:npm/lodash@4.17.20"] = "pkg:npm/lodash@4.17.21"
},
NetVulnerabilityChange = -1
},
RiskAssessment = RemediationRisk.Low,
TestRequirements = new RemediationTestRequirements
{
TestSuites = new[] { "unit", "integration" },
MinCoverage = 80,
RequireAllPass = true,
Timeout = TimeSpan.FromMinutes(15)
},
Authority = RemediationAuthority.Draft,
PrReady = false,
ConfidenceScore = 0.92,
ModelId = "test-model",
GeneratedAt = DateTime.UtcNow.ToString("o"),
InputHashes = new[] { "hash1", "hash2" },
EvidenceRefs = new[] { "evidence/sbom-001", "evidence/vuln-001" }
};
}
#endregion
#region Stub Implementations
private sealed class StubRemediationPlanner : IRemediationPlanner
{
private readonly Dictionary<string, RemediationPlan> _plans = new();
private readonly bool _patchVersionBump;
private readonly bool _majorVersionBump;
private readonly bool _buildWillFail;
private readonly bool _testsWillFail;
private readonly bool _allVerificationsPassed;
private readonly bool _hasBreakingChanges;
public StubRemediationPlanner(
bool patchVersionBump = false,
bool majorVersionBump = false,
bool buildWillFail = false,
bool testsWillFail = false,
bool allVerificationsPassed = false,
bool hasBreakingChanges = false)
{
_patchVersionBump = patchVersionBump;
_majorVersionBump = majorVersionBump;
_buildWillFail = buildWillFail;
_testsWillFail = testsWillFail;
_allVerificationsPassed = allVerificationsPassed;
_hasBreakingChanges = hasBreakingChanges;
}
public Task<RemediationPlan> GeneratePlanAsync(
RemediationPlanRequest request,
CancellationToken cancellationToken = default)
{
var planId = $"plan-{Guid.NewGuid():N}";
var (actionType, risk, confidence) = DetermineStepDetails(request);
var steps = new List<RemediationStep>
{
new()
{
Order = 1,
ActionType = actionType,
FilePath = GetFilePath(request),
Description = $"Fix {request.VulnerabilityId}",
PreviousValue = "old",
NewValue = "new",
Risk = risk
}
};
var authority = DetermineAuthority(request);
var prReady = authority == RemediationAuthority.Verified;
var notReadyReason = GetNotReadyReason();
if (_hasBreakingChanges)
{
confidence *= 0.6;
}
var plan = new RemediationPlan
{
PlanId = planId,
Request = request,
Steps = steps,
ExpectedDelta = new ExpectedSbomDelta
{
Added = Array.Empty<string>(),
Removed = Array.Empty<string>(),
Upgraded = new Dictionary<string, string>
{
[request.ComponentPurl] = request.ComponentPurl + "-fixed"
},
NetVulnerabilityChange = -1
},
RiskAssessment = risk,
TestRequirements = new RemediationTestRequirements
{
TestSuites = new[] { "unit", "integration" },
MinCoverage = 80,
RequireAllPass = true,
Timeout = TimeSpan.FromMinutes(15)
},
Authority = authority,
PrReady = prReady,
NotReadyReason = notReadyReason,
ConfidenceScore = confidence,
ModelId = "stub-model",
GeneratedAt = DateTime.UtcNow.ToString("o"),
InputHashes = new[] { $"input:{request.FindingId}", $"input:{request.ArtifactDigest}" },
EvidenceRefs = new[] { "evidence/ref-001" }
};
_plans[planId] = plan;
return Task.FromResult(plan);
}
private (string ActionType, RemediationRisk Risk, double Confidence) DetermineStepDetails(
RemediationPlanRequest request)
{
var actionType = request.RemediationType switch
{
RemediationType.Bump => "update_package",
RemediationType.Upgrade => "update_base_image",
RemediationType.Config => "update_config",
RemediationType.Backport => "apply_patch",
_ => "update_package"
};
if (_patchVersionBump)
return (actionType, RemediationRisk.Low, 0.95);
if (_majorVersionBump)
return (actionType, RemediationRisk.High, 0.65);
return (actionType, RemediationRisk.Medium, 0.85);
}
private string GetFilePath(RemediationPlanRequest request)
{
if (request.ComponentPurl.StartsWith("pkg:npm"))
return "package.json";
if (request.ComponentPurl.StartsWith("pkg:pypi"))
return "requirements.txt";
if (request.ComponentPurl.StartsWith("pkg:oci"))
return "Dockerfile";
return "package.json";
}
private RemediationAuthority DetermineAuthority(RemediationPlanRequest request)
{
if (!request.AutoCreatePr)
return RemediationAuthority.Draft;
if (_buildWillFail || _testsWillFail)
return RemediationAuthority.Suggestion;
if (_allVerificationsPassed)
return RemediationAuthority.Verified;
return RemediationAuthority.Draft;
}
private string? GetNotReadyReason()
{
if (_buildWillFail)
return "Build failed during verification";
if (_testsWillFail)
return "Tests failed during verification";
return null;
}
public Task<bool> ValidatePlanAsync(string planId, CancellationToken cancellationToken = default)
{
return Task.FromResult(_plans.ContainsKey(planId));
}
public Task<RemediationPlan?> GetPlanAsync(string planId, CancellationToken cancellationToken = default)
{
_plans.TryGetValue(planId, out var plan);
return Task.FromResult(plan);
}
}
private sealed class StubPullRequestGenerator : IPullRequestGenerator
{
private readonly Dictionary<string, PullRequestResult> _prs = new();
private int _prCounter;
public string ScmType => ScmTypeOverride ?? "github";
public string? ScmTypeOverride { get; set; }
public Task<PullRequestResult> CreatePullRequestAsync(
RemediationPlan plan,
CancellationToken cancellationToken = default)
{
var prId = $"pr-{Guid.NewGuid():N}";
_prCounter++;
var branchName = $"stellaops-fix-{plan.Request.VulnerabilityId.ToLowerInvariant()}-{_prCounter}";
var result = new PullRequestResult
{
PrId = prId,
PrNumber = _prCounter,
Url = $"https://github.com/test/repo/pull/{_prCounter}",
BranchName = branchName,
Status = PullRequestStatus.Open,
CreatedAt = DateTime.UtcNow.ToString("o"),
UpdatedAt = DateTime.UtcNow.ToString("o")
};
_prs[prId] = result;
return Task.FromResult(result);
}
public Task<PullRequestResult> GetStatusAsync(string prId, CancellationToken cancellationToken = default)
{
if (!_prs.TryGetValue(prId, out var result))
throw new InvalidOperationException($"PR {prId} not found");
return Task.FromResult(result);
}
public Task UpdateWithDeltaVerdictAsync(
string prId,
DeltaVerdictResult deltaVerdict,
CancellationToken cancellationToken = default)
{
if (!_prs.TryGetValue(prId, out var result))
throw new InvalidOperationException($"PR {prId} not found");
_prs[prId] = result with
{
DeltaVerdict = deltaVerdict,
UpdatedAt = DateTime.UtcNow.ToString("o")
};
return Task.CompletedTask;
}
public Task ClosePullRequestAsync(string prId, string reason, CancellationToken cancellationToken = default)
{
if (!_prs.TryGetValue(prId, out var result))
throw new InvalidOperationException($"PR {prId} not found");
_prs[prId] = result with
{
Status = PullRequestStatus.Closed,
StatusMessage = reason,
UpdatedAt = DateTime.UtcNow.ToString("o")
};
return Task.CompletedTask;
}
}
#endregion
}

View File

@@ -10,11 +10,13 @@ using Microsoft.Extensions.Options;
using StellaOps.AdvisoryAI.Providers;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class SbomContextHttpClientTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GetContextAsync_MapsPayloadToDocument()
{
const string payload = """
@@ -98,7 +100,8 @@ public sealed class SbomContextHttpClientTests
Assert.Contains("includeBlastRadius=true", handler.LastRequest.RequestUri!.Query);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GetContextAsync_ReturnsNullOnNotFound()
{
var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.NotFound));
@@ -110,7 +113,8 @@ public sealed class SbomContextHttpClientTests
Assert.Null(result);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task GetContextAsync_ThrowsForServerError()
{
var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError)

View File

@@ -2,11 +2,13 @@ using FluentAssertions;
using StellaOps.AdvisoryAI.Abstractions;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class SbomContextRequestTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public void Constructor_NormalizesWhitespaceAndLimits()
{
var request = new SbomContextRequest(
@@ -25,7 +27,8 @@ public sealed class SbomContextRequestTests
request.IncludeBlastRadius.Should().BeFalse();
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public void Constructor_AllowsNullPurlAndDefaults()
{
var request = new SbomContextRequest(artifactId: "scan-123", purl: null);

View File

@@ -12,11 +12,13 @@ using StellaOps.AdvisoryAI.Providers;
using StellaOps.AdvisoryAI.Retrievers;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class SbomContextRetrieverTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RetrieveAsync_ReturnsDeterministicOrderingAndMetadata()
{
var document = new SbomContextDocument(
@@ -103,7 +105,8 @@ public sealed class SbomContextRetrieverTests
result.Metadata["blast_radius_present"].Should().Be(bool.TrueString);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RetrieveAsync_ReturnsEmptyWhenNoDocument()
{
var client = new FakeSbomContextClient(null);
@@ -119,7 +122,8 @@ public sealed class SbomContextRetrieverTests
result.BlastRadius.Should().BeNull();
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RetrieveAsync_HonorsEnvironmentFlagToggle()
{
var document = new SbomContextDocument(
@@ -152,7 +156,8 @@ public sealed class SbomContextRetrieverTests
client.LastQuery.IncludeBlastRadius.Should().BeFalse();
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task RetrieveAsync_DeduplicatesDependencyPaths()
{
var duplicatePath = ImmutableArray.Create(

View File

@@ -2,11 +2,13 @@ using FluentAssertions;
using StellaOps.AdvisoryAI.Tools;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class SemanticVersionTests
{
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("1.2.3", 1, 2, 3, false)]
[InlineData("1.2.3-alpha", 1, 2, 3, true)]
[InlineData("0.0.1+build", 0, 0, 1, false)]
@@ -21,7 +23,8 @@ public sealed class SemanticVersionTests
(version.PreRelease.Count > 0).Should().Be(hasPreRelease);
}
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("01.0.0")]
[InlineData("1..0")]
[InlineData("1.0.0-")]
@@ -33,7 +36,8 @@ public sealed class SemanticVersionTests
act.Should().Throw<FormatException>();
}
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("1.2.3", "1.2.3", 0)]
[InlineData("1.2.3", "1.2.4", -1)]
[InlineData("1.3.0", "1.2.9", 1)]
@@ -48,7 +52,8 @@ public sealed class SemanticVersionTests
Math.Sign(leftVersion.CompareTo(rightVersion)).Should().Be(expectedSign);
}
[Theory]
[Trait("Category", TestCategories.Unit)]
[Theory]
[InlineData("1.2.3", ">=1.0.0,<2.0.0", true)]
[InlineData("0.9.0", ">=1.0.0", false)]
[InlineData("1.2.3-beta", ">=1.2.3", false)]
@@ -61,7 +66,8 @@ public sealed class SemanticVersionTests
SemanticVersionRange.Satisfies(version, range).Should().Be(expected);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public void DeterministicToolset_ComparesSemverAndEvr()
{
IDeterministicToolset toolset = new DeterministicToolset();

View File

@@ -19,6 +19,7 @@
<ProjectReference Include="..\..\..\Concelier\__Libraries\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="..\..\..\Concelier\__Libraries\StellaOps.Concelier.RawModels\StellaOps.Concelier.RawModels.csproj" />
<ProjectReference Include="..\..\..\Excititor\__Libraries\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.TestKit\StellaOps.TestKit.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="TestData/*.json">

View File

@@ -11,11 +11,13 @@ using StellaOps.AdvisoryAI.Abstractions;
using StellaOps.AdvisoryAI.Documents;
using Xunit;
using StellaOps.TestKit;
namespace StellaOps.AdvisoryAI.Tests;
public sealed class ToolsetServiceCollectionExtensionsTests
{
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public void AddAdvisoryDeterministicToolset_RegistersSingleton()
{
var services = new ServiceCollection();
@@ -29,7 +31,8 @@ public sealed class ToolsetServiceCollectionExtensionsTests
Assert.Same(toolsetA, toolsetB);
}
[Fact]
[Trait("Category", TestCategories.Unit)]
[Fact]
public void AddAdvisoryPipeline_RegistersOrchestrator()
{
var services = new ServiceCollection();