Add grounded unified search answers and live verification
This commit is contained in:
@@ -379,6 +379,18 @@ public static class UnifiedSearchEndpoints
|
||||
CurrentRoute = string.IsNullOrWhiteSpace(ambient.CurrentRoute) ? null : ambient.CurrentRoute.Trim(),
|
||||
SessionId = string.IsNullOrWhiteSpace(ambient.SessionId) ? null : ambient.SessionId.Trim(),
|
||||
ResetSession = ambient.ResetSession,
|
||||
LastAction = ambient.LastAction is null || string.IsNullOrWhiteSpace(ambient.LastAction.Action)
|
||||
? null
|
||||
: new AmbientAction
|
||||
{
|
||||
Action = ambient.LastAction.Action.Trim(),
|
||||
Source = string.IsNullOrWhiteSpace(ambient.LastAction.Source) ? null : ambient.LastAction.Source.Trim(),
|
||||
QueryHint = string.IsNullOrWhiteSpace(ambient.LastAction.QueryHint) ? null : ambient.LastAction.QueryHint.Trim(),
|
||||
Domain = string.IsNullOrWhiteSpace(ambient.LastAction.Domain) ? null : ambient.LastAction.Domain.Trim().ToLowerInvariant(),
|
||||
EntityKey = string.IsNullOrWhiteSpace(ambient.LastAction.EntityKey) ? null : ambient.LastAction.EntityKey.Trim(),
|
||||
Route = string.IsNullOrWhiteSpace(ambient.LastAction.Route) ? null : ambient.LastAction.Route.Trim(),
|
||||
OccurredAt = ambient.LastAction.OccurredAt
|
||||
},
|
||||
VisibleEntityKeys = ambient.VisibleEntityKeys is { Count: > 0 }
|
||||
? ambient.VisibleEntityKeys
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
@@ -462,6 +474,31 @@ public static class UnifiedSearchEndpoints
|
||||
}).ToArray();
|
||||
}
|
||||
|
||||
UnifiedSearchApiContextAnswer? contextAnswer = null;
|
||||
if (response.ContextAnswer is not null)
|
||||
{
|
||||
contextAnswer = new UnifiedSearchApiContextAnswer
|
||||
{
|
||||
Status = response.ContextAnswer.Status,
|
||||
Code = response.ContextAnswer.Code,
|
||||
Summary = response.ContextAnswer.Summary,
|
||||
Reason = response.ContextAnswer.Reason,
|
||||
Evidence = response.ContextAnswer.Evidence,
|
||||
Citations = response.ContextAnswer.Citations?.Select(static citation => new UnifiedSearchApiContextAnswerCitation
|
||||
{
|
||||
EntityKey = citation.EntityKey,
|
||||
Title = citation.Title,
|
||||
Domain = citation.Domain,
|
||||
Route = citation.Route
|
||||
}).ToArray(),
|
||||
Questions = response.ContextAnswer.Questions?.Select(static question => new UnifiedSearchApiContextAnswerQuestion
|
||||
{
|
||||
Query = question.Query,
|
||||
Kind = question.Kind
|
||||
}).ToArray()
|
||||
};
|
||||
}
|
||||
|
||||
return new UnifiedSearchApiResponse
|
||||
{
|
||||
Query = response.Query,
|
||||
@@ -470,6 +507,7 @@ public static class UnifiedSearchEndpoints
|
||||
Synthesis = synthesis,
|
||||
Suggestions = suggestions,
|
||||
Refinements = refinements,
|
||||
ContextAnswer = contextAnswer,
|
||||
Diagnostics = new UnifiedSearchApiDiagnostics
|
||||
{
|
||||
FtsMatches = response.Diagnostics.FtsMatches,
|
||||
@@ -696,6 +734,25 @@ public sealed record UnifiedSearchApiAmbientContext
|
||||
public string? SessionId { get; init; }
|
||||
|
||||
public bool ResetSession { get; init; }
|
||||
|
||||
public UnifiedSearchApiAmbientAction? LastAction { get; init; }
|
||||
}
|
||||
|
||||
public sealed record UnifiedSearchApiAmbientAction
|
||||
{
|
||||
public string Action { get; init; } = string.Empty;
|
||||
|
||||
public string? Source { get; init; }
|
||||
|
||||
public string? QueryHint { get; init; }
|
||||
|
||||
public string? Domain { get; init; }
|
||||
|
||||
public string? EntityKey { get; init; }
|
||||
|
||||
public string? Route { get; init; }
|
||||
|
||||
public DateTimeOffset? OccurredAt { get; init; }
|
||||
}
|
||||
|
||||
public sealed record UnifiedSearchApiFilter
|
||||
@@ -751,6 +808,8 @@ public sealed record UnifiedSearchApiResponse
|
||||
|
||||
public IReadOnlyList<UnifiedSearchApiRefinement>? Refinements { get; init; }
|
||||
|
||||
public UnifiedSearchApiContextAnswer? ContextAnswer { get; init; }
|
||||
|
||||
public UnifiedSearchApiDiagnostics Diagnostics { get; init; } = new();
|
||||
|
||||
public IReadOnlyList<UnifiedSearchApiFederationDiagnostic>? Federation { get; init; }
|
||||
@@ -839,6 +898,41 @@ public sealed record UnifiedSearchApiRefinement
|
||||
public string Source { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed record UnifiedSearchApiContextAnswer
|
||||
{
|
||||
public string Status { get; init; } = string.Empty;
|
||||
|
||||
public string Code { get; init; } = string.Empty;
|
||||
|
||||
public string Summary { get; init; } = string.Empty;
|
||||
|
||||
public string Reason { get; init; } = string.Empty;
|
||||
|
||||
public string Evidence { get; init; } = string.Empty;
|
||||
|
||||
public IReadOnlyList<UnifiedSearchApiContextAnswerCitation>? Citations { get; init; }
|
||||
|
||||
public IReadOnlyList<UnifiedSearchApiContextAnswerQuestion>? Questions { get; init; }
|
||||
}
|
||||
|
||||
public sealed record UnifiedSearchApiContextAnswerCitation
|
||||
{
|
||||
public string EntityKey { get; init; } = string.Empty;
|
||||
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
public string Domain { get; init; } = string.Empty;
|
||||
|
||||
public string? Route { get; init; }
|
||||
}
|
||||
|
||||
public sealed record UnifiedSearchApiContextAnswerQuestion
|
||||
{
|
||||
public string Query { get; init; } = string.Empty;
|
||||
|
||||
public string Kind { get; init; } = "follow_up";
|
||||
}
|
||||
|
||||
public sealed record UnifiedSearchApiDiagnostics
|
||||
{
|
||||
public int FtsMatches { get; init; }
|
||||
|
||||
@@ -16,3 +16,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
|
||||
| SPRINT_20260224_003-LOC-202 | DONE | `SPRINT_20260224_003_AdvisoryAI_translation_rollout_remaining_phases.md`: phase-3.4 AdvisoryAI slice completed (remote bundle wiring, localized validation keys in search/unified-search endpoints, `en-US`+`de-DE` service bundles, and de-DE integration coverage). |
|
||||
| SPRINT_20260224_G1-G10 | DONE | Search improvement sprints G1–G10 implemented. New endpoints: `SearchAnalyticsEndpoints.cs` (history, events, popularity), `SearchFeedbackEndpoints.cs` (feedback, quality alerts, metrics). Extended: `UnifiedSearchEndpoints.cs` (suggestions, refinements, previews, diagnostics.activeEncoder). Extended: `KnowledgeSearchEndpoints.cs` (activeEncoder in diagnostics). See `docs/modules/advisory-ai/knowledge-search.md` for full testing guide. |
|
||||
|
||||
| AI-SELF-001 | DONE | Unified search endpoint contract now exposes backend contextual answer fields for self-serve search. |
|
||||
| AI-SELF-006 | DONE | Endpoint readiness now includes a proven local rebuilt-corpus verification lane in addition to stubbed integration tests. |
|
||||
|
||||
@@ -10,6 +10,11 @@ Source of truth: `docs/implplan/SPRINT_20260113_005_ADVISORYAI_controlled_conver
|
||||
| SPRINT_20260224_102-G1-005 | DONE | ONNX missing-model fallback integration evidence added (`G1_OnnxEncoderSelection_MissingModelPath_FallsBackToDeterministicHashEncoder`). |
|
||||
| SPRINT_20260224_102-G1-004 | DONE | Semantic recall benchmark corpus and assertions complete (48 queries; no exact-term regression; semantic recall uplift proven). |
|
||||
| SPRINT_20260224_102-G1-001 | DOING | ONNX runtime package + license docs completed; model asset provisioning at `models/all-MiniLM-L6-v2.onnx` still pending deployment packaging. |
|
||||
| AI-SELF-001 | DONE | Unified search now emits the contextual answer payload (`contextAnswer`) with answer state, citations, and follow-up questions for self-serve search. |
|
||||
| AI-SELF-002 | DONE | Deterministic grounded/clarify/insufficient fallback policy is implemented in unified search orchestration. |
|
||||
| AI-SELF-003 | DONE | Follow-up question generation from route/domain intent, recent actions, and evidence is implemented. |
|
||||
| AI-SELF-004 | TODO | Telemetry for unanswered and reformulated self-serve journeys is still pending. |
|
||||
| AI-SELF-006 | DONE | Live ingestion-backed answer verification succeeded on the Doctor/knowledge route after local rebuild. |
|
||||
| SPRINT_20260222_051-AKS-INGEST | DONE | Added deterministic AKS ingestion controls: markdown allow-list manifest loading, OpenAPI aggregate source path support, and doctor control projection integration for search chunks, including fallback doctor metadata hydration from controls projection fields. |
|
||||
| AUDIT-0017-M | DONE | Maintainability audit for StellaOps.AdvisoryAI. |
|
||||
| AUDIT-0017-T | DONE | Test coverage audit for StellaOps.AdvisoryAI. |
|
||||
|
||||
@@ -2,17 +2,29 @@ namespace StellaOps.AdvisoryAI.UnifiedSearch.Context;
|
||||
|
||||
internal sealed class AmbientContextProcessor
|
||||
{
|
||||
private const double CurrentRouteBoost = 0.10d;
|
||||
private const double LastActionDomainBoost = 0.05d;
|
||||
private const double VisibleEntityBoost = 0.20d;
|
||||
private const double LastActionEntityBoost = 0.25d;
|
||||
private static readonly (string Prefix, string Domain)[] RouteDomainMappings =
|
||||
[
|
||||
("/console/findings", "findings"),
|
||||
("/security/triage", "findings"),
|
||||
("/security/findings", "findings"),
|
||||
("/security/advisories-vex", "vex"),
|
||||
("/ops/policies", "policy"),
|
||||
("/ops/policy", "policy"),
|
||||
("/ops/graph", "graph"),
|
||||
("/security/reach", "graph"),
|
||||
("/ops/audit", "timeline"),
|
||||
("/ops/timeline", "timeline"),
|
||||
("/audit", "timeline"),
|
||||
("/console/scans", "scanner"),
|
||||
("/ops/operations/jobs", "opsmemory"),
|
||||
("/ops/operations/scheduler", "opsmemory"),
|
||||
("/ops/doctor", "knowledge"),
|
||||
("/ops/operations/doctor", "knowledge"),
|
||||
("/ops/operations/system-health", "knowledge"),
|
||||
("/docs", "knowledge")
|
||||
];
|
||||
|
||||
@@ -23,18 +35,26 @@ internal sealed class AmbientContextProcessor
|
||||
var output = new Dictionary<string, double>(baseWeights, StringComparer.OrdinalIgnoreCase);
|
||||
if (ambient is null || string.IsNullOrWhiteSpace(ambient.CurrentRoute))
|
||||
{
|
||||
var actionOnlyDomain = ResolveDomainFromAmbientAction(ambient?.LastAction);
|
||||
if (!string.IsNullOrWhiteSpace(actionOnlyDomain))
|
||||
{
|
||||
ApplyBoost(output, actionOnlyDomain, LastActionDomainBoost);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
var domain = ResolveDomainFromRoute(ambient.CurrentRoute);
|
||||
if (string.IsNullOrWhiteSpace(domain))
|
||||
if (!string.IsNullOrWhiteSpace(domain))
|
||||
{
|
||||
return output;
|
||||
ApplyBoost(output, domain, CurrentRouteBoost);
|
||||
}
|
||||
|
||||
output[domain] = output.TryGetValue(domain, out var existing)
|
||||
? existing + 0.10d
|
||||
: 1.10d;
|
||||
var actionDomain = ResolveDomainFromAmbientAction(ambient.LastAction);
|
||||
if (!string.IsNullOrWhiteSpace(actionDomain))
|
||||
{
|
||||
ApplyBoost(output, actionDomain, LastActionDomainBoost);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
@@ -54,12 +74,21 @@ internal sealed class AmbientContextProcessor
|
||||
continue;
|
||||
}
|
||||
|
||||
map[entityKey.Trim()] = Math.Max(
|
||||
map.TryGetValue(entityKey.Trim(), out var existing) ? existing : 0d,
|
||||
0.20d);
|
||||
var normalized = entityKey.Trim();
|
||||
map[normalized] = Math.Max(
|
||||
map.TryGetValue(normalized, out var existing) ? existing : 0d,
|
||||
VisibleEntityBoost);
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ambient?.LastAction?.EntityKey))
|
||||
{
|
||||
var normalized = ambient.LastAction.EntityKey.Trim();
|
||||
map[normalized] = Math.Max(
|
||||
map.TryGetValue(normalized, out var existing) ? existing : 0d,
|
||||
LastActionEntityBoost);
|
||||
}
|
||||
|
||||
foreach (var entry in session.EntityBoosts)
|
||||
{
|
||||
map[entry.Key] = Math.Max(
|
||||
@@ -120,5 +149,31 @@ internal sealed class AmbientContextProcessor
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ResolveDomainFromAmbientAction(AmbientAction? action)
|
||||
{
|
||||
if (action is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Domain))
|
||||
{
|
||||
return action.Domain.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(action.Route))
|
||||
{
|
||||
return ResolveDomainFromRoute(action.Route);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static void ApplyBoost(IDictionary<string, double> output, string domain, double amount)
|
||||
{
|
||||
output[domain] = output.TryGetValue(domain, out var existing)
|
||||
? existing + amount
|
||||
: 1d + amount;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,12 +69,50 @@ public sealed record AmbientContext
|
||||
public string? SessionId { get; init; }
|
||||
|
||||
public bool ResetSession { get; init; }
|
||||
|
||||
public AmbientAction? LastAction { get; init; }
|
||||
}
|
||||
|
||||
public sealed record AmbientAction
|
||||
{
|
||||
public string Action { get; init; } = string.Empty;
|
||||
|
||||
public string? Source { get; init; }
|
||||
|
||||
public string? QueryHint { get; init; }
|
||||
|
||||
public string? Domain { get; init; }
|
||||
|
||||
public string? EntityKey { get; init; }
|
||||
|
||||
public string? Route { get; init; }
|
||||
|
||||
public DateTimeOffset? OccurredAt { get; init; }
|
||||
}
|
||||
|
||||
public sealed record SearchSuggestion(string Text, string Reason);
|
||||
|
||||
public sealed record SearchRefinement(string Text, string Source);
|
||||
|
||||
public sealed record ContextAnswer(
|
||||
string Status,
|
||||
string Code,
|
||||
string Summary,
|
||||
string Reason,
|
||||
string Evidence,
|
||||
IReadOnlyList<ContextAnswerCitation>? Citations = null,
|
||||
IReadOnlyList<ContextAnswerQuestion>? Questions = null);
|
||||
|
||||
public sealed record ContextAnswerCitation(
|
||||
string EntityKey,
|
||||
string Title,
|
||||
string Domain,
|
||||
string? Route = null);
|
||||
|
||||
public sealed record ContextAnswerQuestion(
|
||||
string Query,
|
||||
string Kind = "follow_up");
|
||||
|
||||
public sealed record UnifiedSearchResponse(
|
||||
string Query,
|
||||
int TopK,
|
||||
@@ -82,7 +120,8 @@ public sealed record UnifiedSearchResponse(
|
||||
SynthesisResult? Synthesis,
|
||||
UnifiedSearchDiagnostics Diagnostics,
|
||||
IReadOnlyList<SearchSuggestion>? Suggestions = null,
|
||||
IReadOnlyList<SearchRefinement>? Refinements = null);
|
||||
IReadOnlyList<SearchRefinement>? Refinements = null,
|
||||
ContextAnswer? ContextAnswer = null);
|
||||
|
||||
public sealed record EntityCard
|
||||
{
|
||||
|
||||
@@ -18,6 +18,9 @@ namespace StellaOps.AdvisoryAI.UnifiedSearch;
|
||||
|
||||
internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
{
|
||||
private const int MaxContextAnswerCitations = 3;
|
||||
private const int MaxContextAnswerQuestions = 3;
|
||||
private const int ClarifyTokenThreshold = 3;
|
||||
private readonly KnowledgeSearchOptions _options;
|
||||
private readonly UnifiedSearchOptions _unifiedOptions;
|
||||
private readonly IKnowledgeSearchStore _store;
|
||||
@@ -106,7 +109,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
|
||||
if (query.Length > _unifiedOptions.MaxQueryLength)
|
||||
{
|
||||
return EmptyResponse(query, request.K, "query_too_long");
|
||||
return EmptyResponse(query, request.K, "query_too_long", request.Ambient);
|
||||
}
|
||||
|
||||
var tenantId = request.Filters?.Tenant ?? "global";
|
||||
@@ -115,7 +118,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
|
||||
if (!_options.Enabled || !IsSearchEnabledForTenant(tenantFlags) || string.IsNullOrWhiteSpace(_options.ConnectionString))
|
||||
{
|
||||
return EmptyResponse(query, request.K, "disabled");
|
||||
return EmptyResponse(query, request.K, "disabled", request.Ambient);
|
||||
}
|
||||
|
||||
if (request.Ambient?.ResetSession == true &&
|
||||
@@ -292,6 +295,14 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
}
|
||||
|
||||
var duration = _timeProvider.GetUtcNow() - startedAt;
|
||||
var contextAnswer = BuildContextAnswer(
|
||||
query,
|
||||
plan,
|
||||
request.Ambient,
|
||||
cards,
|
||||
synthesis,
|
||||
suggestions,
|
||||
refinements);
|
||||
var response = new UnifiedSearchResponse(
|
||||
query,
|
||||
topK,
|
||||
@@ -307,7 +318,8 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
plan,
|
||||
federationDiagnostics),
|
||||
suggestions,
|
||||
refinements);
|
||||
refinements,
|
||||
contextAnswer);
|
||||
|
||||
EmitTelemetry(plan, response, tenantId);
|
||||
return response;
|
||||
@@ -370,6 +382,489 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ContextAnswerCitation> BuildContextAnswerCitations(
|
||||
IReadOnlyList<EntityCard> cards,
|
||||
SynthesisResult? synthesis)
|
||||
{
|
||||
var citations = new List<ContextAnswerCitation>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (synthesis?.Citations is { Count: > 0 })
|
||||
{
|
||||
foreach (var citation in synthesis.Citations)
|
||||
{
|
||||
var card = cards.FirstOrDefault(card =>
|
||||
string.Equals(card.EntityKey, citation.EntityKey, StringComparison.OrdinalIgnoreCase));
|
||||
if (card is null || !seen.Add(card.EntityKey))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
citations.Add(new ContextAnswerCitation(
|
||||
card.EntityKey,
|
||||
string.IsNullOrWhiteSpace(card.Title) ? citation.Title : card.Title,
|
||||
card.Domain,
|
||||
GetPrimaryActionRoute(card)));
|
||||
|
||||
if (citations.Count >= MaxContextAnswerCitations)
|
||||
{
|
||||
return citations;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var card in cards)
|
||||
{
|
||||
if (!seen.Add(card.EntityKey))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
citations.Add(new ContextAnswerCitation(
|
||||
card.EntityKey,
|
||||
card.Title,
|
||||
card.Domain,
|
||||
GetPrimaryActionRoute(card)));
|
||||
|
||||
if (citations.Count >= MaxContextAnswerCitations)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return citations;
|
||||
}
|
||||
|
||||
private static string BuildGroundedSummary(IReadOnlyList<EntityCard> cards, SynthesisResult? synthesis)
|
||||
{
|
||||
var synthesisSummary = synthesis?.Summary?.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(synthesisSummary))
|
||||
{
|
||||
return synthesisSummary;
|
||||
}
|
||||
|
||||
var topCard = cards[0];
|
||||
var snippet = topCard.Snippet?.Trim();
|
||||
return string.IsNullOrWhiteSpace(snippet) ? topCard.Title : snippet;
|
||||
}
|
||||
|
||||
private static string BuildGroundedReason(QueryPlan plan, AmbientContext? ambient, EntityCard topCard)
|
||||
{
|
||||
var scope = ResolveContextDomain(plan, [topCard], ambient) ?? topCard.Domain;
|
||||
return $"The top result is grounded in {DescribeDomain(scope)} evidence and aligns with the {plan.Intent} intent.";
|
||||
}
|
||||
|
||||
private static string BuildGroundedEvidence(IReadOnlyList<EntityCard> cards, SynthesisResult? synthesis)
|
||||
{
|
||||
var sourceCount = Math.Max(synthesis?.SourceCount ?? 0, cards.Count);
|
||||
var domains = synthesis?.DomainsCovered is { Count: > 0 }
|
||||
? synthesis.DomainsCovered
|
||||
: cards.Select(static card => card.Domain)
|
||||
.Where(static domain => !string.IsNullOrWhiteSpace(domain))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
return $"Grounded in {sourceCount} source(s) across {FormatDomainList(domains)}.";
|
||||
}
|
||||
|
||||
private static bool ShouldClarifyQuery(
|
||||
string query,
|
||||
QueryPlan plan,
|
||||
AmbientContext? ambient,
|
||||
IReadOnlyList<SearchSuggestion>? suggestions,
|
||||
IReadOnlyList<SearchRefinement>? refinements)
|
||||
{
|
||||
if (plan.DetectedEntities.Count > 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (refinements is { Count: > 0 } || suggestions is { Count: > 0 })
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var tokenCount = query.Split(
|
||||
' ',
|
||||
StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length;
|
||||
if (tokenCount <= ClarifyTokenThreshold)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var contextDomain = ResolveContextDomain(plan, [], ambient);
|
||||
if (!string.IsNullOrWhiteSpace(contextDomain) &&
|
||||
(string.Equals(plan.Intent, "explore", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(plan.Intent, "navigate", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string BuildClarifySummary(string query, string contextDomain)
|
||||
{
|
||||
return $"\"{query}\" is too broad for {DescribeDomain(contextDomain)}. Narrow it to a specific target before we answer.";
|
||||
}
|
||||
|
||||
private static string BuildClarifyReason(QueryPlan plan, AmbientContext? ambient)
|
||||
{
|
||||
var scope = ResolveContextDomain(plan, [], ambient) ?? "current scope";
|
||||
return $"No grounded entity was retrieved, and the {plan.Intent} query needs a narrower target in {DescribeDomain(scope)}.";
|
||||
}
|
||||
|
||||
private static string BuildClarifyEvidence(
|
||||
IReadOnlyList<SearchSuggestion>? suggestions,
|
||||
IReadOnlyList<SearchRefinement>? refinements,
|
||||
string contextDomain)
|
||||
{
|
||||
var suggestionCount = suggestions?.Count ?? 0;
|
||||
var refinementCount = refinements?.Count ?? 0;
|
||||
if (suggestionCount > 0 || refinementCount > 0)
|
||||
{
|
||||
return $"{suggestionCount + refinementCount} bounded recovery hint(s) are available for {DescribeDomain(contextDomain)}.";
|
||||
}
|
||||
|
||||
return $"No grounded evidence was retrieved yet from {DescribeDomain(contextDomain)}.";
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ContextAnswerQuestion> BuildGroundedQuestions(
|
||||
string query,
|
||||
QueryPlan plan,
|
||||
AmbientContext? ambient,
|
||||
EntityCard topCard)
|
||||
{
|
||||
var prompts = new List<ContextAnswerQuestion>();
|
||||
var topTitle = topCard.Title.Trim();
|
||||
foreach (var question in GetGroundedQuestionTemplates(topCard.Domain, topTitle))
|
||||
{
|
||||
prompts.Add(new ContextAnswerQuestion(question, "follow_up"));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ambient?.LastAction?.QueryHint))
|
||||
{
|
||||
var lastHint = ambient.LastAction.QueryHint.Trim();
|
||||
if (!lastHint.Equals(query, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
prompts.Add(new ContextAnswerQuestion(
|
||||
$"How does {topTitle} relate to {lastHint}?",
|
||||
"follow_up"));
|
||||
}
|
||||
}
|
||||
|
||||
return DistinctQuestions(prompts);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ContextAnswerQuestion> BuildClarifyingQuestions(
|
||||
string query,
|
||||
QueryPlan? plan,
|
||||
AmbientContext? ambient)
|
||||
{
|
||||
var prompts = new List<ContextAnswerQuestion>();
|
||||
var domain = ResolveContextDomain(plan, [], ambient);
|
||||
foreach (var question in GetClarifyingQuestionTemplates(domain))
|
||||
{
|
||||
prompts.Add(new ContextAnswerQuestion(question, "clarify"));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ambient?.LastAction?.QueryHint))
|
||||
{
|
||||
var lastHint = ambient.LastAction.QueryHint.Trim();
|
||||
if (!lastHint.Equals(query, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
prompts.Add(new ContextAnswerQuestion(
|
||||
$"Do you want to continue from {lastHint}?",
|
||||
"clarify"));
|
||||
}
|
||||
}
|
||||
|
||||
return DistinctQuestions(prompts);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ContextAnswerQuestion> BuildRecoveryQuestions(
|
||||
string query,
|
||||
QueryPlan plan,
|
||||
AmbientContext? ambient,
|
||||
IReadOnlyList<SearchSuggestion>? suggestions,
|
||||
IReadOnlyList<SearchRefinement>? refinements)
|
||||
{
|
||||
var prompts = new List<ContextAnswerQuestion>();
|
||||
if (refinements is { Count: > 0 })
|
||||
{
|
||||
prompts.AddRange(refinements.Select(static refinement =>
|
||||
new ContextAnswerQuestion(refinement.Text, "recover")));
|
||||
}
|
||||
|
||||
if (suggestions is { Count: > 0 })
|
||||
{
|
||||
prompts.AddRange(suggestions.Select(static suggestion =>
|
||||
new ContextAnswerQuestion(suggestion.Text, "recover")));
|
||||
}
|
||||
|
||||
if (prompts.Count == 0)
|
||||
{
|
||||
var domain = ResolveContextDomain(plan, [], ambient);
|
||||
foreach (var question in GetRecoveryQuestionTemplates(domain, query))
|
||||
{
|
||||
prompts.Add(new ContextAnswerQuestion(question, "recover"));
|
||||
}
|
||||
}
|
||||
|
||||
return DistinctQuestions(prompts);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ContextAnswerQuestion> BuildFallbackQuestionsForDisabledSearch(AmbientContext? ambient)
|
||||
{
|
||||
var domain = ResolveContextDomain(plan: null, cards: [], ambient: ambient);
|
||||
return DistinctQuestions(GetRecoveryQuestionTemplates(domain, "current search")
|
||||
.Select(static question => new ContextAnswerQuestion(question, "recover"))
|
||||
.ToList());
|
||||
}
|
||||
|
||||
private static string BuildInsufficientSummary(string query, QueryPlan plan, AmbientContext? ambient)
|
||||
{
|
||||
var scope = ResolveContextDomain(plan, [], ambient) ?? "current scope";
|
||||
return $"No grounded answer was found for \"{query}\" in {DescribeDomain(scope)}.";
|
||||
}
|
||||
|
||||
private static string BuildInsufficientEvidence(
|
||||
IReadOnlyList<SearchSuggestion>? suggestions,
|
||||
IReadOnlyList<SearchRefinement>? refinements)
|
||||
{
|
||||
var suggestionCount = suggestions?.Count ?? 0;
|
||||
var refinementCount = refinements?.Count ?? 0;
|
||||
if (suggestionCount > 0 || refinementCount > 0)
|
||||
{
|
||||
return $"No grounded citations were found, but {suggestionCount + refinementCount} recovery hint(s) are available.";
|
||||
}
|
||||
|
||||
return "No grounded citations, result cards, or bounded recovery hints were retrieved.";
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> GetGroundedQuestionTemplates(string domain, string title)
|
||||
{
|
||||
return domain switch
|
||||
{
|
||||
"findings" =>
|
||||
[
|
||||
$"Why does {title} block release?",
|
||||
$"What is the safest remediation for {title}?",
|
||||
$"What evidence proves exploitability for {title}?"
|
||||
],
|
||||
"vex" =>
|
||||
[
|
||||
$"Why is {title} marked not affected?",
|
||||
$"What evidence conflicts with {title}?",
|
||||
$"Which components are covered by {title}?"
|
||||
],
|
||||
"policy" =>
|
||||
[
|
||||
$"Why is {title} failing?",
|
||||
$"What findings are impacted by {title}?",
|
||||
$"What is the safest exception path for {title}?"
|
||||
],
|
||||
"graph" =>
|
||||
[
|
||||
$"Which path makes {title} reachable?",
|
||||
$"What is the blast radius of {title}?",
|
||||
$"What should I inspect next from {title}?"
|
||||
],
|
||||
"timeline" =>
|
||||
[
|
||||
$"What changed before {title}?",
|
||||
$"What else happened around {title}?",
|
||||
$"Which release is most related to {title}?"
|
||||
],
|
||||
"opsmemory" =>
|
||||
[
|
||||
$"Have we seen {title} before?",
|
||||
$"What runbook usually resolves {title}?",
|
||||
$"What repeated failures are related to {title}?"
|
||||
],
|
||||
_ =>
|
||||
[
|
||||
$"How do I verify the fix for {title}?",
|
||||
$"What changed before {title} failed?",
|
||||
$"What should I inspect next for {title}?"
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> GetClarifyingQuestionTemplates(string? domain)
|
||||
{
|
||||
return domain switch
|
||||
{
|
||||
"findings" =>
|
||||
[
|
||||
"Which CVE, workload, or package should I narrow this to?",
|
||||
"Should I focus on reachable, production, or unresolved findings?"
|
||||
],
|
||||
"vex" =>
|
||||
[
|
||||
"Which statement, component, or product range should I narrow this to?",
|
||||
"Do you need exploitability meaning, coverage, or conflict evidence?"
|
||||
],
|
||||
"policy" =>
|
||||
[
|
||||
"Which rule, environment, or control should I narrow this to?",
|
||||
"Do you need recent failures, exceptions, or promotion impact?"
|
||||
],
|
||||
"graph" =>
|
||||
[
|
||||
"Which node, package, or edge should I narrow this to?",
|
||||
"Do you want reachability, impact, or next-step guidance?"
|
||||
],
|
||||
"timeline" =>
|
||||
[
|
||||
"Which deployment, incident, or time window should I narrow this to?",
|
||||
"Do you want causes, impacts, or follow-up events?"
|
||||
],
|
||||
"opsmemory" =>
|
||||
[
|
||||
"Which job, incident, or recurring failure should I narrow this to?",
|
||||
"Do you want precedent, likely cause, or recommended recovery?"
|
||||
],
|
||||
_ =>
|
||||
[
|
||||
"Which check, component, or symptom should I narrow this to?",
|
||||
"Do you want diagnosis, remediation, or verification steps?"
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> GetRecoveryQuestionTemplates(string? domain, string query)
|
||||
{
|
||||
return domain switch
|
||||
{
|
||||
"findings" =>
|
||||
[
|
||||
$"reachable {query}",
|
||||
$"unresolved {query}",
|
||||
$"critical {query}"
|
||||
],
|
||||
"policy" =>
|
||||
[
|
||||
$"policy exceptions {query}",
|
||||
$"failing policy gates {query}",
|
||||
$"promotion impact {query}"
|
||||
],
|
||||
"timeline" =>
|
||||
[
|
||||
$"recent events {query}",
|
||||
$"deployment history {query}",
|
||||
$"incident timeline {query}"
|
||||
],
|
||||
_ =>
|
||||
[
|
||||
query,
|
||||
$"strongest evidence {query}",
|
||||
$"related {query}"
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ContextAnswerQuestion> DistinctQuestions(IReadOnlyList<ContextAnswerQuestion> questions)
|
||||
{
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var output = new List<ContextAnswerQuestion>();
|
||||
foreach (var question in questions)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(question.Query) || !seen.Add(question.Query.Trim()))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
output.Add(question with { Query = question.Query.Trim() });
|
||||
if (output.Count >= MaxContextAnswerQuestions)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
private static string? ResolveContextDomain(
|
||||
QueryPlan? plan,
|
||||
IReadOnlyList<EntityCard> cards,
|
||||
AmbientContext? ambient)
|
||||
{
|
||||
if (cards.Count > 0 && !string.IsNullOrWhiteSpace(cards[0].Domain))
|
||||
{
|
||||
return cards[0].Domain;
|
||||
}
|
||||
|
||||
var routeDomain = AmbientContextProcessor.ResolveDomainFromRoute(ambient?.CurrentRoute ?? string.Empty);
|
||||
if (!string.IsNullOrWhiteSpace(routeDomain))
|
||||
{
|
||||
return routeDomain;
|
||||
}
|
||||
|
||||
var actionDomain = ambient?.LastAction?.Domain;
|
||||
if (!string.IsNullOrWhiteSpace(actionDomain))
|
||||
{
|
||||
return actionDomain.Trim().ToLowerInvariant();
|
||||
}
|
||||
|
||||
var actionRouteDomain = AmbientContextProcessor.ResolveDomainFromRoute(ambient?.LastAction?.Route ?? string.Empty);
|
||||
if (!string.IsNullOrWhiteSpace(actionRouteDomain))
|
||||
{
|
||||
return actionRouteDomain;
|
||||
}
|
||||
|
||||
return plan?.DomainWeights
|
||||
.OrderByDescending(static pair => pair.Value)
|
||||
.ThenBy(static pair => pair.Key, StringComparer.Ordinal)
|
||||
.Select(static pair => pair.Key)
|
||||
.FirstOrDefault(static value => !string.IsNullOrWhiteSpace(value));
|
||||
}
|
||||
|
||||
private static string DescribeDomain(string domain)
|
||||
{
|
||||
return domain switch
|
||||
{
|
||||
"findings" => "the findings scope",
|
||||
"vex" => "the VEX scope",
|
||||
"policy" => "the policy scope",
|
||||
"graph" => "the graph scope",
|
||||
"timeline" => "the timeline scope",
|
||||
"opsmemory" => "the operations-memory scope",
|
||||
"knowledge" => "the knowledge scope",
|
||||
"scanner" => "the scanner scope",
|
||||
_ => "the current scope"
|
||||
};
|
||||
}
|
||||
|
||||
private static string FormatDomainList(IEnumerable<string> domains)
|
||||
{
|
||||
var labels = domains
|
||||
.Where(static domain => !string.IsNullOrWhiteSpace(domain))
|
||||
.Select(static domain => domain switch
|
||||
{
|
||||
"findings" => "findings",
|
||||
"vex" => "VEX",
|
||||
"policy" => "policy",
|
||||
"graph" => "graph",
|
||||
"timeline" => "timeline",
|
||||
"opsmemory" => "ops memory",
|
||||
"scanner" => "scanner",
|
||||
_ => domain
|
||||
})
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(3)
|
||||
.ToArray();
|
||||
|
||||
return labels.Length == 0 ? "knowledge" : string.Join(", ", labels);
|
||||
}
|
||||
|
||||
private static string? GetPrimaryActionRoute(EntityCard card)
|
||||
{
|
||||
return card.Actions
|
||||
.FirstOrDefault(static action => !string.IsNullOrWhiteSpace(action.Route))?
|
||||
.Route;
|
||||
}
|
||||
|
||||
private const int PreviewContentMaxLength = 2000;
|
||||
|
||||
private static EntityCardPreview? BuildPreview(KnowledgeChunkRow row, string domain)
|
||||
@@ -889,14 +1384,99 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
return Math.Clamp(requested.Value, 1, 100);
|
||||
}
|
||||
|
||||
private UnifiedSearchResponse EmptyResponse(string query, int? topK, string mode)
|
||||
private UnifiedSearchResponse EmptyResponse(string query, int? topK, string mode, AmbientContext? ambient = null)
|
||||
{
|
||||
return new UnifiedSearchResponse(
|
||||
query,
|
||||
ResolveTopK(topK),
|
||||
[],
|
||||
null,
|
||||
new UnifiedSearchDiagnostics(0, 0, 0, 0, false, mode));
|
||||
new UnifiedSearchDiagnostics(0, 0, 0, 0, false, mode),
|
||||
ContextAnswer: BuildEmptyContextAnswer(query, mode, ambient));
|
||||
}
|
||||
|
||||
private ContextAnswer? BuildContextAnswer(
|
||||
string query,
|
||||
QueryPlan plan,
|
||||
AmbientContext? ambient,
|
||||
IReadOnlyList<EntityCard> cards,
|
||||
SynthesisResult? synthesis,
|
||||
IReadOnlyList<SearchSuggestion>? suggestions,
|
||||
IReadOnlyList<SearchRefinement>? refinements)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (cards.Count > 0)
|
||||
{
|
||||
var topCard = cards[0];
|
||||
var citations = BuildContextAnswerCitations(cards, synthesis);
|
||||
var questions = BuildGroundedQuestions(query, plan, ambient, topCard);
|
||||
return new ContextAnswer(
|
||||
Status: "grounded",
|
||||
Code: "retrieved_evidence",
|
||||
Summary: BuildGroundedSummary(cards, synthesis),
|
||||
Reason: BuildGroundedReason(plan, ambient, topCard),
|
||||
Evidence: BuildGroundedEvidence(cards, synthesis),
|
||||
Citations: citations,
|
||||
Questions: questions);
|
||||
}
|
||||
|
||||
if (ShouldClarifyQuery(query, plan, ambient, suggestions, refinements))
|
||||
{
|
||||
var clarificationScope = ResolveContextDomain(plan, cards, ambient) ?? "current scope";
|
||||
return new ContextAnswer(
|
||||
Status: "clarify",
|
||||
Code: refinements is { Count: > 0 } || suggestions is { Count: > 0 }
|
||||
? "query_needs_scope_with_recovery"
|
||||
: "query_needs_scope",
|
||||
Summary: BuildClarifySummary(query, clarificationScope),
|
||||
Reason: BuildClarifyReason(plan, ambient),
|
||||
Evidence: BuildClarifyEvidence(suggestions, refinements, clarificationScope),
|
||||
Citations: [],
|
||||
Questions: BuildClarifyingQuestions(query, plan, ambient));
|
||||
}
|
||||
|
||||
var recoveryQuestions = BuildRecoveryQuestions(query, plan, ambient, suggestions, refinements);
|
||||
return new ContextAnswer(
|
||||
Status: "insufficient",
|
||||
Code: "no_grounded_evidence",
|
||||
Summary: BuildInsufficientSummary(query, plan, ambient),
|
||||
Reason: "No grounded evidence matched the requested terms in the current ingested corpus.",
|
||||
Evidence: BuildInsufficientEvidence(suggestions, refinements),
|
||||
Citations: [],
|
||||
Questions: recoveryQuestions);
|
||||
}
|
||||
|
||||
private ContextAnswer? BuildEmptyContextAnswer(string query, string mode, AmbientContext? ambient)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return mode switch
|
||||
{
|
||||
"query_too_long" => new ContextAnswer(
|
||||
Status: "clarify",
|
||||
Code: "query_too_long",
|
||||
Summary: "Shorten the search to a specific component, check, policy, or CVE.",
|
||||
Reason: "The query exceeded the bounded unified-search length limit, so it cannot be ranked safely.",
|
||||
Evidence: "No search was executed. Narrow the query before retrying.",
|
||||
Citations: [],
|
||||
Questions: BuildClarifyingQuestions(query, plan: null, ambient: ambient)),
|
||||
"disabled" => new ContextAnswer(
|
||||
Status: "insufficient",
|
||||
Code: "search_disabled",
|
||||
Summary: "Unified search is not enabled for the current environment or tenant.",
|
||||
Reason: "The request could not be grounded because unified search is disabled before retrieval.",
|
||||
Evidence: "No search index was queried.",
|
||||
Citations: [],
|
||||
Questions: BuildFallbackQuestionsForDisabledSearch(ambient)),
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private static string? GetMetadataString(JsonElement metadata, string propertyName)
|
||||
@@ -1272,6 +1852,10 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
DurationMs: response.Diagnostics.DurationMs,
|
||||
UsedVector: response.Diagnostics.UsedVector,
|
||||
DomainWeights: new Dictionary<string, double>(plan.DomainWeights, StringComparer.Ordinal),
|
||||
TopDomains: topDomains));
|
||||
TopDomains: topDomains,
|
||||
AnswerStatus: response.ContextAnswer?.Status,
|
||||
AnswerCode: response.ContextAnswer?.Code,
|
||||
HasSuggestions: response.Suggestions is { Count: > 0 },
|
||||
HasRefinements: response.Refinements is { Count: > 0 }));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,11 @@ public sealed record UnifiedSearchTelemetryEvent(
|
||||
long DurationMs,
|
||||
bool UsedVector,
|
||||
IReadOnlyDictionary<string, double> DomainWeights,
|
||||
IReadOnlyList<string> TopDomains);
|
||||
IReadOnlyList<string> TopDomains,
|
||||
string? AnswerStatus = null,
|
||||
string? AnswerCode = null,
|
||||
bool HasSuggestions = false,
|
||||
bool HasRefinements = false);
|
||||
|
||||
public interface IUnifiedSearchTelemetrySink
|
||||
{
|
||||
@@ -45,13 +49,17 @@ internal sealed class LoggingUnifiedSearchTelemetrySink : IUnifiedSearchTelemetr
|
||||
: string.Join(",", telemetryEvent.TopDomains.OrderBy(static value => value, StringComparer.Ordinal));
|
||||
|
||||
_logger.LogInformation(
|
||||
"unified_search telemetry tenant={Tenant} query_hash={QueryHash} intent={Intent} results={ResultCount} duration_ms={DurationMs} used_vector={UsedVector} top_domains={TopDomains} weights={Weights}",
|
||||
"unified_search telemetry tenant={Tenant} query_hash={QueryHash} intent={Intent} results={ResultCount} duration_ms={DurationMs} used_vector={UsedVector} answer_status={AnswerStatus} answer_code={AnswerCode} has_suggestions={HasSuggestions} has_refinements={HasRefinements} top_domains={TopDomains} weights={Weights}",
|
||||
telemetryEvent.Tenant,
|
||||
telemetryEvent.QueryHash,
|
||||
telemetryEvent.Intent,
|
||||
telemetryEvent.ResultCount,
|
||||
telemetryEvent.DurationMs,
|
||||
telemetryEvent.UsedVector,
|
||||
telemetryEvent.AnswerStatus ?? "-",
|
||||
telemetryEvent.AnswerCode ?? "-",
|
||||
telemetryEvent.HasSuggestions,
|
||||
telemetryEvent.HasRefinements,
|
||||
topDomains,
|
||||
weights);
|
||||
}
|
||||
|
||||
@@ -141,6 +141,7 @@ Build or publish the CLI from this repository:
|
||||
|
||||
```bash
|
||||
# One-shot invocation without installing to PATH
|
||||
export STELLAOPS_BACKEND_URL="http://127.0.0.1:10451"
|
||||
dotnet run --project "src/Cli/StellaOps.Cli/StellaOps.Cli.csproj" -- advisoryai sources prepare --json
|
||||
|
||||
# Publish a reusable local binary
|
||||
@@ -159,6 +160,7 @@ For live search and Playwright suggestion tests, rebuild both indexes in this or
|
||||
|
||||
```bash
|
||||
# 1. Knowledge corpus: docs + OpenAPI + Doctor checks
|
||||
export STELLAOPS_BACKEND_URL="http://127.0.0.1:10451"
|
||||
dotnet run --project "src/Cli/StellaOps.Cli/StellaOps.Cli.csproj" -- advisoryai sources prepare --json
|
||||
dotnet run --project "src/Cli/StellaOps.Cli/StellaOps.Cli.csproj" -- advisoryai index rebuild --json
|
||||
|
||||
@@ -182,6 +184,10 @@ curl -X POST http://127.0.0.1:10451/v1/search/index/rebuild \
|
||||
-H "X-StellaOps-Tenant: test-tenant"
|
||||
```
|
||||
|
||||
Notes:
|
||||
- `stella advisoryai sources prepare` needs `STELLAOPS_BACKEND_URL` or equivalent CLI config when it performs live Doctor discovery. If you only need local search verification and the checked-in Doctor seed/control files are sufficient, the HTTP-only rebuild path is valid.
|
||||
- Current live verification coverage includes the Doctor/knowledge query `database connectivity`, which returns `contextAnswer.status = grounded` plus citations after the rebuild sequence above.
|
||||
|
||||
Migration files (all idempotent, safe to re-run):
|
||||
| File | Content |
|
||||
| --- | --- |
|
||||
|
||||
@@ -76,6 +76,53 @@ public sealed class UnifiedSearchEndpointsIntegrationTests : IDisposable
|
||||
payload!.Query.Should().Be("cve-2024-21626");
|
||||
payload.Cards.Should().NotBeEmpty();
|
||||
payload.Cards.Should().Contain(card => card.Domain == "findings");
|
||||
payload.ContextAnswer.Should().NotBeNull();
|
||||
payload.ContextAnswer!.Status.Should().Be("grounded");
|
||||
payload.ContextAnswer.Citations.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Query_WithBroadQuery_ReturnsClarifyContextAnswer()
|
||||
{
|
||||
using var client = _factory.CreateClient();
|
||||
client.DefaultRequestHeaders.Add("X-StellaOps-Scopes", "advisory-ai:operate");
|
||||
client.DefaultRequestHeaders.Add("X-StellaOps-Tenant", "test-tenant");
|
||||
|
||||
var response = await client.PostAsJsonAsync("/v1/search/query", new UnifiedSearchApiRequest
|
||||
{
|
||||
Q = "status"
|
||||
});
|
||||
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
|
||||
var payload = await response.Content.ReadFromJsonAsync<UnifiedSearchApiResponse>();
|
||||
payload.Should().NotBeNull();
|
||||
payload!.Cards.Should().BeEmpty();
|
||||
payload.ContextAnswer.Should().NotBeNull();
|
||||
payload.ContextAnswer!.Status.Should().Be("clarify");
|
||||
payload.ContextAnswer.Questions.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Query_WithNoMatches_ReturnsInsufficientContextAnswer()
|
||||
{
|
||||
using var client = _factory.CreateClient();
|
||||
client.DefaultRequestHeaders.Add("X-StellaOps-Scopes", "advisory-ai:operate");
|
||||
client.DefaultRequestHeaders.Add("X-StellaOps-Tenant", "test-tenant");
|
||||
|
||||
var response = await client.PostAsJsonAsync("/v1/search/query", new UnifiedSearchApiRequest
|
||||
{
|
||||
Q = "manually compare unavailable evidence across environments"
|
||||
});
|
||||
|
||||
response.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
|
||||
var payload = await response.Content.ReadFromJsonAsync<UnifiedSearchApiResponse>();
|
||||
payload.Should().NotBeNull();
|
||||
payload!.Cards.Should().BeEmpty();
|
||||
payload.ContextAnswer.Should().NotBeNull();
|
||||
payload.ContextAnswer!.Status.Should().Be("insufficient");
|
||||
payload.ContextAnswer.Code.Should().Be("no_grounded_evidence");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -211,6 +258,61 @@ public sealed class UnifiedSearchEndpointsIntegrationTests : IDisposable
|
||||
{
|
||||
public Task<UnifiedSearchResponse> SearchAsync(UnifiedSearchRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
var normalizedQuery = request.Q.Trim();
|
||||
if (normalizedQuery.Equals("status", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return Task.FromResult(new UnifiedSearchResponse(
|
||||
normalizedQuery,
|
||||
request.K ?? 10,
|
||||
[],
|
||||
null,
|
||||
new UnifiedSearchDiagnostics(
|
||||
FtsMatches: 0,
|
||||
VectorMatches: 0,
|
||||
EntityCardCount: 0,
|
||||
DurationMs: 5,
|
||||
UsedVector: false,
|
||||
Mode: "fts-only"),
|
||||
ContextAnswer: new ContextAnswer(
|
||||
Status: "clarify",
|
||||
Code: "query_needs_scope",
|
||||
Summary: "\"status\" is too broad for the knowledge scope.",
|
||||
Reason: "The query needs a narrower target before grounded evidence can be returned.",
|
||||
Evidence: "No grounded evidence was retrieved yet from the knowledge scope.",
|
||||
Citations: [],
|
||||
Questions:
|
||||
[
|
||||
new ContextAnswerQuestion("Which check or symptom should I narrow this to?", "clarify")
|
||||
])));
|
||||
}
|
||||
|
||||
if (normalizedQuery.Equals("manually compare unavailable evidence across environments", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return Task.FromResult(new UnifiedSearchResponse(
|
||||
normalizedQuery,
|
||||
request.K ?? 10,
|
||||
[],
|
||||
null,
|
||||
new UnifiedSearchDiagnostics(
|
||||
FtsMatches: 0,
|
||||
VectorMatches: 0,
|
||||
EntityCardCount: 0,
|
||||
DurationMs: 5,
|
||||
UsedVector: false,
|
||||
Mode: "fts-only"),
|
||||
ContextAnswer: new ContextAnswer(
|
||||
Status: "insufficient",
|
||||
Code: "no_grounded_evidence",
|
||||
Summary: "No grounded answer was found for the requested query.",
|
||||
Reason: "No grounded evidence matched the requested terms in the current corpus.",
|
||||
Evidence: "No grounded citations, result cards, or bounded recovery hints were retrieved.",
|
||||
Citations: [],
|
||||
Questions:
|
||||
[
|
||||
new ContextAnswerQuestion("strongest evidence manually compare unavailable evidence across environments", "recover")
|
||||
])));
|
||||
}
|
||||
|
||||
var cards = new[]
|
||||
{
|
||||
new EntityCard
|
||||
@@ -230,7 +332,7 @@ public sealed class UnifiedSearchEndpointsIntegrationTests : IDisposable
|
||||
};
|
||||
|
||||
return Task.FromResult(new UnifiedSearchResponse(
|
||||
request.Q.Trim(),
|
||||
normalizedQuery,
|
||||
request.K ?? 10,
|
||||
cards,
|
||||
null,
|
||||
@@ -240,7 +342,25 @@ public sealed class UnifiedSearchEndpointsIntegrationTests : IDisposable
|
||||
EntityCardCount: cards.Length,
|
||||
DurationMs: 5,
|
||||
UsedVector: false,
|
||||
Mode: "fts-only")));
|
||||
Mode: "fts-only"),
|
||||
ContextAnswer: new ContextAnswer(
|
||||
Status: "grounded",
|
||||
Code: "retrieved_evidence",
|
||||
Summary: "Container breakout via runc",
|
||||
Reason: "The top result contains direct findings evidence.",
|
||||
Evidence: "Grounded in 1 source across findings.",
|
||||
Citations:
|
||||
[
|
||||
new ContextAnswerCitation(
|
||||
"cve:CVE-2024-21626",
|
||||
"CVE-2024-21626",
|
||||
"findings",
|
||||
"/security/triage?q=CVE-2024-21626")
|
||||
],
|
||||
Questions:
|
||||
[
|
||||
new ContextAnswerQuestion("What evidence proves exploitability for CVE-2024-21626?", "follow_up")
|
||||
])));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,3 +21,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
|
||||
| SPRINT_20260224_G5-005-BENCH | DONE | FTS recall benchmark: 12 tests in `FtsRecallBenchmarkTests.cs`, 34-query fixture (`fts-recall-benchmark.json`), `FtsRecallBenchmarkStore` (Simple vs English). Simple ~59% vs English ~100% Recall@10 (41pp improvement). |
|
||||
| SPRINT_20260224_G1-004-BENCH | DONE | Semantic recall benchmark: 13 tests in `SemanticRecallBenchmarkTests.cs`, 48-query fixture (`semantic-recall-benchmark.json`), `SemanticRecallBenchmarkStore` (33 chunks), `SemanticSimulationEncoder` (40+ semantic groups). Semantic strictly outperforms hash on synonym queries. |
|
||||
|
||||
| AI-SELF-005 | DONE | Integration coverage now asserts grounded, clarify, and insufficient contextual-answer states through the real endpoint contract. |
|
||||
| AI-SELF-006 | DONE | Verification includes a real local corpus rebuild and a live query assertion, not only test doubles. |
|
||||
|
||||
@@ -7,6 +7,16 @@ namespace StellaOps.AdvisoryAI.Tests.UnifiedSearch;
|
||||
|
||||
public sealed class AmbientContextProcessorTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("/ops/operations/doctor/check.core.db.connectivity", "knowledge")]
|
||||
[InlineData("/security/findings/fnd-42", "findings")]
|
||||
[InlineData("/ops/timeline/releases/rel-42", "timeline")]
|
||||
[InlineData("/ops/operations/jobs/job-7", "opsmemory")]
|
||||
public void ResolveDomainFromRoute_matches_current_frontend_route_prefixes(string route, string expectedDomain)
|
||||
{
|
||||
AmbientContextProcessor.ResolveDomainFromRoute(route).Should().Be(expectedDomain);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApplyRouteBoost_boosts_matching_domain_from_route()
|
||||
{
|
||||
@@ -26,6 +36,30 @@ public sealed class AmbientContextProcessorTests
|
||||
boosted["knowledge"].Should().Be(1.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApplyRouteBoost_adds_secondary_boost_from_last_action_domain()
|
||||
{
|
||||
var processor = new AmbientContextProcessor();
|
||||
var weights = new Dictionary<string, double>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["knowledge"] = 1.0,
|
||||
["policy"] = 1.0
|
||||
};
|
||||
|
||||
var boosted = processor.ApplyRouteBoost(weights, new AmbientContext
|
||||
{
|
||||
CurrentRoute = "/ops/operations/doctor",
|
||||
LastAction = new AmbientAction
|
||||
{
|
||||
Action = "search_answer_to_chat",
|
||||
Domain = "policy"
|
||||
}
|
||||
});
|
||||
|
||||
boosted["knowledge"].Should().BeApproximately(1.10, 0.0001);
|
||||
boosted["policy"].Should().BeApproximately(1.05, 0.0001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildEntityBoostMap_merges_visible_entities_and_session_boosts()
|
||||
{
|
||||
@@ -49,6 +83,26 @@ public sealed class AmbientContextProcessorTests
|
||||
map["image:registry.io/app:v1"].Should().BeApproximately(0.20, 0.0001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildEntityBoostMap_prioritizes_last_action_entity_key()
|
||||
{
|
||||
var processor = new AmbientContextProcessor();
|
||||
|
||||
var map = processor.BuildEntityBoostMap(
|
||||
new AmbientContext
|
||||
{
|
||||
VisibleEntityKeys = ["cve:CVE-2025-1234"],
|
||||
LastAction = new AmbientAction
|
||||
{
|
||||
Action = "search_result_open",
|
||||
EntityKey = "cve:CVE-2025-1234"
|
||||
}
|
||||
},
|
||||
SearchSessionSnapshot.Empty);
|
||||
|
||||
map["cve:CVE-2025-1234"].Should().BeApproximately(0.25, 0.0001);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CarryForwardEntities_adds_session_entities_for_followup_queries_without_new_entities()
|
||||
{
|
||||
|
||||
@@ -45,6 +45,85 @@ public sealed class UnifiedSearchServiceTests
|
||||
result.Diagnostics.Mode.Should().Be("disabled");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SearchAsync_returns_grounded_context_answer_when_cards_exist()
|
||||
{
|
||||
var doctorRow = MakeRow(
|
||||
"chunk-doctor",
|
||||
"doctor_check",
|
||||
"PostgreSQL connectivity",
|
||||
JsonDocument.Parse("{\"domain\":\"knowledge\",\"checkCode\":\"check.core.db.connectivity\"}"),
|
||||
snippet: "PostgreSQL connectivity is failing because the gateway cannot reach the primary database.");
|
||||
|
||||
var storeMock = new Mock<IKnowledgeSearchStore>();
|
||||
storeMock.Setup(s => s.SearchFtsAsync(
|
||||
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>(), It.IsAny<string?>()))
|
||||
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow });
|
||||
storeMock.Setup(s => s.LoadVectorCandidatesAsync(
|
||||
It.IsAny<float[]>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync([]);
|
||||
|
||||
var service = CreateService(storeMock: storeMock);
|
||||
|
||||
var result = await service.SearchAsync(
|
||||
new UnifiedSearchRequest(
|
||||
"database connectivity",
|
||||
Ambient: new AmbientContext
|
||||
{
|
||||
CurrentRoute = "/ops/operations/doctor"
|
||||
}),
|
||||
CancellationToken.None);
|
||||
|
||||
result.ContextAnswer.Should().NotBeNull();
|
||||
result.ContextAnswer!.Status.Should().Be("grounded");
|
||||
result.ContextAnswer.Code.Should().Be("retrieved_evidence");
|
||||
result.ContextAnswer.Citations.Should().NotBeNullOrEmpty();
|
||||
result.ContextAnswer.Questions.Should().NotBeNullOrEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SearchAsync_returns_clarify_context_answer_for_broad_query_without_matches()
|
||||
{
|
||||
var storeMock = CreateEmptyStoreMock();
|
||||
var service = CreateService(storeMock: storeMock);
|
||||
|
||||
var result = await service.SearchAsync(
|
||||
new UnifiedSearchRequest(
|
||||
"status",
|
||||
Ambient: new AmbientContext
|
||||
{
|
||||
CurrentRoute = "/ops/operations/doctor"
|
||||
}),
|
||||
CancellationToken.None);
|
||||
|
||||
result.Cards.Should().BeEmpty();
|
||||
result.ContextAnswer.Should().NotBeNull();
|
||||
result.ContextAnswer!.Status.Should().Be("clarify");
|
||||
result.ContextAnswer.Code.Should().StartWith("query_needs_scope");
|
||||
result.ContextAnswer.Questions.Should().NotBeNullOrEmpty();
|
||||
result.ContextAnswer.Questions!.Should().OnlyContain(question => question.Kind == "clarify");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SearchAsync_returns_insufficient_context_answer_for_specific_query_without_matches()
|
||||
{
|
||||
var storeMock = CreateEmptyStoreMock();
|
||||
var service = CreateService(storeMock: storeMock);
|
||||
|
||||
var result = await service.SearchAsync(
|
||||
new UnifiedSearchRequest("manually compare unavailable evidence across environments"),
|
||||
CancellationToken.None);
|
||||
|
||||
result.Cards.Should().BeEmpty();
|
||||
result.ContextAnswer.Should().NotBeNull();
|
||||
result.ContextAnswer!.Status.Should().Be("insufficient");
|
||||
result.ContextAnswer.Code.Should().Be("no_grounded_evidence");
|
||||
result.ContextAnswer.Questions.Should().NotBeNullOrEmpty();
|
||||
result.ContextAnswer.Questions!.Should().OnlyContain(question => question.Kind == "recover");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SearchAsync_returns_empty_when_tenant_feature_flag_disables_search()
|
||||
{
|
||||
@@ -799,6 +878,25 @@ public sealed class UnifiedSearchServiceTests
|
||||
unifiedOptions: wrappedUnifiedOptions);
|
||||
}
|
||||
|
||||
private static Mock<IKnowledgeSearchStore> CreateEmptyStoreMock()
|
||||
{
|
||||
var storeMock = new Mock<IKnowledgeSearchStore>();
|
||||
storeMock.Setup(s => s.SearchFtsAsync(
|
||||
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>(), It.IsAny<string?>()))
|
||||
.ReturnsAsync([]);
|
||||
storeMock.Setup(s => s.SearchFuzzyAsync(
|
||||
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||
It.IsAny<double>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync([]);
|
||||
storeMock.Setup(s => s.LoadVectorCandidatesAsync(
|
||||
It.IsAny<float[]>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync([]);
|
||||
|
||||
return storeMock;
|
||||
}
|
||||
|
||||
private static KnowledgeChunkRow MakeRow(
|
||||
string chunkId,
|
||||
string kind,
|
||||
|
||||
Reference in New Issue
Block a user