Implement MongoDB-based storage for Pack Run approval, artifact, log, and state management
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Added MongoPackRunApprovalStore for managing approval states with MongoDB. - Introduced MongoPackRunArtifactUploader for uploading and storing artifacts. - Created MongoPackRunLogStore to handle logging of pack run events. - Developed MongoPackRunStateStore for persisting and retrieving pack run states. - Implemented unit tests for MongoDB stores to ensure correct functionality. - Added MongoTaskRunnerTestContext for setting up MongoDB test environment. - Enhanced PackRunStateFactory to correctly initialize state with gate reasons.
This commit is contained in:
@@ -1,4 +1,7 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using StellaOps.AdvisoryAI.Guardrails;
|
||||
using StellaOps.AdvisoryAI.Outputs;
|
||||
using StellaOps.AdvisoryAI.Orchestration;
|
||||
@@ -53,27 +56,72 @@ internal sealed class AdvisoryPipelineExecutor : IAdvisoryPipelineExecutor
|
||||
|
||||
var prompt = await _promptAssembler.AssembleAsync(plan, cancellationToken).ConfigureAwait(false);
|
||||
var guardrailResult = await _guardrailPipeline.EvaluateAsync(prompt, cancellationToken).ConfigureAwait(false);
|
||||
var violationCount = guardrailResult.Violations.Length;
|
||||
|
||||
if (guardrailResult.Blocked)
|
||||
{
|
||||
_logger?.LogWarning(
|
||||
"Guardrail blocked advisory pipeline output for {TaskType} on advisory {AdvisoryKey}",
|
||||
"Guardrail blocked advisory pipeline output for {TaskType} on advisory {AdvisoryKey} with {ViolationCount} violations",
|
||||
plan.Request.TaskType,
|
||||
plan.Request.AdvisoryKey,
|
||||
violationCount);
|
||||
}
|
||||
else if (violationCount > 0)
|
||||
{
|
||||
_logger?.LogInformation(
|
||||
"Guardrail recorded {ViolationCount} advisory validation violations for {TaskType} on advisory {AdvisoryKey}",
|
||||
violationCount,
|
||||
plan.Request.TaskType,
|
||||
plan.Request.AdvisoryKey);
|
||||
}
|
||||
|
||||
var citationCoverage = CalculateCitationCoverage(plan, prompt);
|
||||
var activity = Activity.Current;
|
||||
activity?.SetTag("advisory.guardrail_blocked", guardrailResult.Blocked);
|
||||
activity?.SetTag("advisory.validation_failures", violationCount);
|
||||
activity?.SetTag("advisory.citation_coverage", citationCoverage);
|
||||
_metrics.RecordGuardrailOutcome(plan.Request.TaskType, guardrailResult.Blocked, violationCount);
|
||||
_metrics.RecordCitationCoverage(
|
||||
plan.Request.TaskType,
|
||||
citationCoverage,
|
||||
prompt.Citations.Length,
|
||||
plan.StructuredChunks.Length);
|
||||
|
||||
var generatedAt = _timeProvider.GetUtcNow();
|
||||
var output = AdvisoryPipelineOutput.Create(plan, prompt, guardrailResult, generatedAt, planFromCache);
|
||||
await _outputStore.SaveAsync(output, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_metrics.RecordGuardrailResult(plan.Request.TaskType, guardrailResult.Blocked);
|
||||
_metrics.RecordOutputStored(plan.Request.TaskType, planFromCache, guardrailResult.Blocked);
|
||||
|
||||
_logger?.LogInformation(
|
||||
"Stored advisory pipeline output {CacheKey} (task {TaskType}, cache:{CacheHit}, guardrail_blocked:{Blocked})",
|
||||
"Stored advisory pipeline output {CacheKey} (task {TaskType}, cache:{CacheHit}, guardrail_blocked:{Blocked}, validation_failures:{ValidationFailures}, citation_coverage:{CitationCoverage:0.00})",
|
||||
output.CacheKey,
|
||||
plan.Request.TaskType,
|
||||
planFromCache,
|
||||
guardrailResult.Blocked);
|
||||
guardrailResult.Blocked,
|
||||
violationCount,
|
||||
citationCoverage);
|
||||
}
|
||||
|
||||
private static double CalculateCitationCoverage(AdvisoryTaskPlan plan, AdvisoryPrompt prompt)
|
||||
{
|
||||
var structuredCount = plan.StructuredChunks.Length;
|
||||
if (structuredCount <= 0)
|
||||
{
|
||||
return 0d;
|
||||
}
|
||||
|
||||
if (prompt.Citations.IsDefaultOrEmpty)
|
||||
{
|
||||
return 0d;
|
||||
}
|
||||
|
||||
var uniqueCitations = prompt.Citations
|
||||
.Select(citation => (citation.DocumentId, citation.ChunkId))
|
||||
.Distinct()
|
||||
.Count();
|
||||
|
||||
var coverage = (double)uniqueCitations / structuredCount;
|
||||
return Math.Clamp(coverage, 0d, 1d);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user