feat: Update Sprint 110 documentation and enhance Advisory AI tests for determinism and mTLS validation
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
This commit is contained in:
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Linq;
|
||||
using FluentAssertions;
|
||||
using StellaOps.AdvisoryAI.Abstractions;
|
||||
using StellaOps.AdvisoryAI.Documents;
|
||||
@@ -118,6 +119,44 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
|
||||
Math.Abs(measurement.Value - 1d) < 0.0001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExecuteAsync_ComputesPartialCitationCoverage()
|
||||
{
|
||||
using var listener = new MeterListener();
|
||||
var doubleMeasurements = new List<(string Name, double Value, IEnumerable<KeyValuePair<string, object?>> Tags)>();
|
||||
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Meter.Name == AdvisoryPipelineMetrics.MeterName)
|
||||
{
|
||||
l.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
|
||||
listener.SetMeasurementEventCallback<double>((instrument, measurement, tags, state) =>
|
||||
{
|
||||
doubleMeasurements.Add((instrument.Name, measurement, tags));
|
||||
});
|
||||
|
||||
listener.Start();
|
||||
|
||||
var plan = BuildPlanWithTwoChunks(cacheKey: "CACHE-4");
|
||||
var assembler = new PartialCitationPromptAssembler(plan, citationsToKeep: 1);
|
||||
var guardrail = new StubGuardrailPipeline(blocked: false);
|
||||
var store = new InMemoryAdvisoryOutputStore();
|
||||
using var metrics = new AdvisoryPipelineMetrics(_meterFactory);
|
||||
var executor = new AdvisoryPipelineExecutor(assembler, guardrail, store, metrics, TimeProvider.System);
|
||||
|
||||
var message = new AdvisoryTaskQueueMessage(plan.CacheKey, plan.Request);
|
||||
await executor.ExecuteAsync(plan, message, planFromCache: false, CancellationToken.None);
|
||||
|
||||
listener.Dispose();
|
||||
|
||||
doubleMeasurements.Should().Contain(measurement =>
|
||||
measurement.Name == "advisory_ai_citation_coverage_ratio" &&
|
||||
Math.Abs(measurement.Value - 0.5d) < 0.0001);
|
||||
}
|
||||
|
||||
private static AdvisoryTaskPlan BuildMinimalPlan(string cacheKey)
|
||||
{
|
||||
var request = new AdvisoryTaskRequest(
|
||||
@@ -148,6 +187,46 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
|
||||
return plan;
|
||||
}
|
||||
|
||||
private static AdvisoryTaskPlan BuildPlanWithTwoChunks(string cacheKey)
|
||||
{
|
||||
var request = new AdvisoryTaskRequest(
|
||||
AdvisoryTaskType.Summary,
|
||||
advisoryKey: "adv-key",
|
||||
artifactId: "artifact-1",
|
||||
profile: "default");
|
||||
|
||||
var chunkOne = AdvisoryChunk.Create(
|
||||
"doc-1",
|
||||
"chunk-1",
|
||||
"Summary",
|
||||
"para-1",
|
||||
"Summary details",
|
||||
new Dictionary<string, string> { ["section"] = "Summary" });
|
||||
|
||||
var chunkTwo = AdvisoryChunk.Create(
|
||||
"doc-1",
|
||||
"chunk-2",
|
||||
"Impact",
|
||||
"para-2",
|
||||
"Impact details",
|
||||
new Dictionary<string, string> { ["section"] = "Impact" });
|
||||
|
||||
var structured = ImmutableArray.Create(chunkOne, chunkTwo);
|
||||
|
||||
var plan = new AdvisoryTaskPlan(
|
||||
request,
|
||||
cacheKey,
|
||||
promptTemplate: "prompts/advisory/summary.liquid",
|
||||
structuredChunks: structured,
|
||||
vectorResults: ImmutableArray<AdvisoryVectorResult>.Empty,
|
||||
sbomContext: null,
|
||||
dependencyAnalysis: DependencyAnalysisResult.Empty("artifact-1"),
|
||||
budget: new AdvisoryTaskBudget { PromptTokens = 512, CompletionTokens = 256 },
|
||||
metadata: ImmutableDictionary<string, string>.Empty.Add("advisory_key", "adv-key"));
|
||||
|
||||
return plan;
|
||||
}
|
||||
|
||||
private sealed class StubPromptAssembler : IAdvisoryPromptAssembler
|
||||
{
|
||||
public Task<AdvisoryPrompt> AssembleAsync(AdvisoryTaskPlan plan, CancellationToken cancellationToken)
|
||||
@@ -166,6 +245,33 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PartialCitationPromptAssembler : IAdvisoryPromptAssembler
|
||||
{
|
||||
private readonly ImmutableArray<AdvisoryPromptCitation> _citations;
|
||||
|
||||
public PartialCitationPromptAssembler(AdvisoryTaskPlan plan, int citationsToKeep)
|
||||
{
|
||||
_citations = plan.StructuredChunks
|
||||
.Take(Math.Clamp(citationsToKeep, 0, plan.StructuredChunks.Length))
|
||||
.Select((chunk, index) => new AdvisoryPromptCitation(index + 1, chunk.DocumentId, chunk.ChunkId))
|
||||
.ToImmutableArray();
|
||||
}
|
||||
|
||||
public Task<AdvisoryPrompt> AssembleAsync(AdvisoryTaskPlan plan, CancellationToken cancellationToken)
|
||||
{
|
||||
var metadata = ImmutableDictionary<string, string>.Empty.Add("advisory_key", plan.Request.AdvisoryKey);
|
||||
var diagnostics = ImmutableDictionary<string, string>.Empty.Add("structured_chunks", plan.StructuredChunks.Length.ToString());
|
||||
return Task.FromResult(new AdvisoryPrompt(
|
||||
plan.CacheKey,
|
||||
plan.Request.TaskType,
|
||||
plan.Request.Profile,
|
||||
"{\"prompt\":\"value\"}",
|
||||
_citations,
|
||||
metadata,
|
||||
diagnostics));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class StubGuardrailPipeline : IAdvisoryGuardrailPipeline
|
||||
{
|
||||
private readonly AdvisoryGuardrailResult _result;
|
||||
|
||||
@@ -48,6 +48,26 @@ public sealed class AdvisoryPlanCacheTests
|
||||
retrieved.Should().BeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SetAsync_ReplacesPlanAndRefreshesExpiration()
|
||||
{
|
||||
var timeProvider = new FakeTimeProvider(DateTimeOffset.UtcNow);
|
||||
var cache = CreateCache(timeProvider, ttl: TimeSpan.FromMinutes(1));
|
||||
var original = CreatePlan(cacheKey: "stable-cache", advisoryKey: "ADV-123");
|
||||
await cache.SetAsync(original.CacheKey, original, CancellationToken.None);
|
||||
|
||||
timeProvider.Advance(TimeSpan.FromSeconds(45));
|
||||
|
||||
var replacement = CreatePlan(cacheKey: "stable-cache", advisoryKey: "ADV-999");
|
||||
await cache.SetAsync(replacement.CacheKey, replacement, CancellationToken.None);
|
||||
|
||||
timeProvider.Advance(TimeSpan.FromSeconds(50)); // total 95s since first insert, 50s since replacement
|
||||
|
||||
var retrieved = await cache.TryGetAsync(replacement.CacheKey, CancellationToken.None);
|
||||
retrieved.Should().NotBeNull();
|
||||
retrieved!.Request.AdvisoryKey.Should().Be("ADV-999");
|
||||
}
|
||||
|
||||
private static InMemoryAdvisoryPlanCache CreateCache(FakeTimeProvider timeProvider, TimeSpan? ttl = null)
|
||||
{
|
||||
var options = Options.Create(new AdvisoryPlanCacheOptions
|
||||
@@ -59,9 +79,9 @@ public sealed class AdvisoryPlanCacheTests
|
||||
return new InMemoryAdvisoryPlanCache(options, timeProvider);
|
||||
}
|
||||
|
||||
private static AdvisoryTaskPlan CreatePlan()
|
||||
private static AdvisoryTaskPlan CreatePlan(string cacheKey = "plan-cache-key", string advisoryKey = "ADV-123")
|
||||
{
|
||||
var request = new AdvisoryTaskRequest(AdvisoryTaskType.Summary, "ADV-123", artifactId: "artifact-1");
|
||||
var request = new AdvisoryTaskRequest(AdvisoryTaskType.Summary, advisoryKey, artifactId: "artifact-1");
|
||||
var chunk = AdvisoryChunk.Create("doc-1", "chunk-1", "section", "para", "text");
|
||||
var structured = ImmutableArray.Create(chunk);
|
||||
var vectors = ImmutableArray.Create(new AdvisoryVectorResult("query", ImmutableArray<VectorRetrievalMatch>.Empty));
|
||||
@@ -72,7 +92,7 @@ public sealed class AdvisoryPlanCacheTests
|
||||
new KeyValuePair<string, string>("task_type", request.TaskType.ToString())
|
||||
});
|
||||
|
||||
return new AdvisoryTaskPlan(request, "plan-cache-key", "template", structured, vectors, sbom, dependency, new AdvisoryTaskBudget(), metadata);
|
||||
return new AdvisoryTaskPlan(request, cacheKey, "template", structured, vectors, sbom, dependency, new AdvisoryTaskBudget(), metadata);
|
||||
}
|
||||
|
||||
private sealed class FakeTimeProvider : TimeProvider
|
||||
|
||||
Reference in New Issue
Block a user