|
|
|
|
@@ -21,6 +21,11 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
private const int MaxContextAnswerCitations = 3;
|
|
|
|
|
private const int MaxContextAnswerQuestions = 3;
|
|
|
|
|
private const int ClarifyTokenThreshold = 3;
|
|
|
|
|
private const int MaxSuggestionViabilityQueries = 6;
|
|
|
|
|
private const int MaxOverflowCards = 4;
|
|
|
|
|
private const int CoverageCandidateWindow = 24;
|
|
|
|
|
private const double OverflowScoreBandRatio = 0.15d;
|
|
|
|
|
private const double BlendedAnswerScoreBandRatio = 0.12d;
|
|
|
|
|
private readonly KnowledgeSearchOptions _options;
|
|
|
|
|
private readonly UnifiedSearchOptions _unifiedOptions;
|
|
|
|
|
private readonly IKnowledgeSearchStore _store;
|
|
|
|
|
@@ -57,6 +62,12 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
// Refinement threshold: only suggest when result count is below this (G10-004)
|
|
|
|
|
private const int RefinementResultThreshold = 3;
|
|
|
|
|
|
|
|
|
|
private enum SearchExecutionKind
|
|
|
|
|
{
|
|
|
|
|
UserVisible,
|
|
|
|
|
SuggestionPreflight
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public UnifiedSearchService(
|
|
|
|
|
IOptions<KnowledgeSearchOptions> options,
|
|
|
|
|
IKnowledgeSearchStore store,
|
|
|
|
|
@@ -96,7 +107,77 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
_telemetrySink = telemetrySink;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<UnifiedSearchResponse> SearchAsync(UnifiedSearchRequest request, CancellationToken cancellationToken)
|
|
|
|
|
public Task<UnifiedSearchResponse> SearchAsync(UnifiedSearchRequest request, CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
return SearchAsyncInternal(request, cancellationToken, SearchExecutionKind.UserVisible);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<SearchSuggestionViabilityResponse> EvaluateSuggestionsAsync(
|
|
|
|
|
SearchSuggestionViabilityRequest request,
|
|
|
|
|
CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
|
|
|
|
|
|
|
|
var normalizedQueries = request.Queries
|
|
|
|
|
.Where(static query => !string.IsNullOrWhiteSpace(query))
|
|
|
|
|
.Select(static query => KnowledgeSearchText.NormalizeWhitespace(query))
|
|
|
|
|
.Where(static query => !string.IsNullOrWhiteSpace(query))
|
|
|
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
|
|
|
.Take(MaxSuggestionViabilityQueries)
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
var fallbackCoverage = new UnifiedSearchCoverage(
|
|
|
|
|
CurrentScopeDomain: ResolveAmbientScopeDomain(request.Ambient),
|
|
|
|
|
CurrentScopeWeighted: !string.IsNullOrWhiteSpace(request.Ambient?.CurrentRoute),
|
|
|
|
|
Domains: []);
|
|
|
|
|
|
|
|
|
|
if (normalizedQueries.Length == 0)
|
|
|
|
|
{
|
|
|
|
|
return new SearchSuggestionViabilityResponse([], fallbackCoverage);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var results = new List<SearchSuggestionViabilityResult>(normalizedQueries.Length);
|
|
|
|
|
UnifiedSearchCoverage? aggregateCoverage = null;
|
|
|
|
|
|
|
|
|
|
foreach (var query in normalizedQueries)
|
|
|
|
|
{
|
|
|
|
|
var response = await SearchAsyncInternal(
|
|
|
|
|
new UnifiedSearchRequest(
|
|
|
|
|
query,
|
|
|
|
|
K: Math.Min(5, _unifiedOptions.MaxCards),
|
|
|
|
|
Filters: request.Filters,
|
|
|
|
|
IncludeSynthesis: false,
|
|
|
|
|
IncludeDebug: false,
|
|
|
|
|
Ambient: request.Ambient),
|
|
|
|
|
cancellationToken,
|
|
|
|
|
SearchExecutionKind.SuggestionPreflight).ConfigureAwait(false);
|
|
|
|
|
|
|
|
|
|
aggregateCoverage = MergeCoverage(aggregateCoverage, response.Coverage);
|
|
|
|
|
var cardCount = response.Cards.Count + (response.Overflow?.Cards.Count ?? 0);
|
|
|
|
|
var answer = response.ContextAnswer;
|
|
|
|
|
|
|
|
|
|
results.Add(new SearchSuggestionViabilityResult(
|
|
|
|
|
Query: query,
|
|
|
|
|
Viable: cardCount > 0 || string.Equals(answer?.Status, "clarify", StringComparison.OrdinalIgnoreCase),
|
|
|
|
|
Status: answer?.Status ?? "insufficient",
|
|
|
|
|
Code: answer?.Code ?? "no_grounded_evidence",
|
|
|
|
|
CardCount: cardCount,
|
|
|
|
|
LeadingDomain: response.Cards.FirstOrDefault()?.Domain
|
|
|
|
|
?? response.Overflow?.Cards.FirstOrDefault()?.Domain
|
|
|
|
|
?? response.Coverage?.Domains.FirstOrDefault(static domain => domain.HasVisibleResults)?.Domain
|
|
|
|
|
?? response.Coverage?.CurrentScopeDomain,
|
|
|
|
|
Reason: answer?.Reason ?? "No grounded evidence matched the suggestion in the active corpus."));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return new SearchSuggestionViabilityResponse(
|
|
|
|
|
results,
|
|
|
|
|
aggregateCoverage ?? fallbackCoverage);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async Task<UnifiedSearchResponse> SearchAsyncInternal(
|
|
|
|
|
UnifiedSearchRequest request,
|
|
|
|
|
CancellationToken cancellationToken,
|
|
|
|
|
SearchExecutionKind executionKind)
|
|
|
|
|
{
|
|
|
|
|
ArgumentNullException.ThrowIfNull(request);
|
|
|
|
|
|
|
|
|
|
@@ -107,13 +188,17 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
return EmptyResponse(string.Empty, request.K, "empty");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var emitObservability = executionKind == SearchExecutionKind.UserVisible;
|
|
|
|
|
var tenantId = request.Filters?.Tenant ?? "global";
|
|
|
|
|
var userId = request.Filters?.UserId ?? "anonymous";
|
|
|
|
|
|
|
|
|
|
if (query.Length > _unifiedOptions.MaxQueryLength)
|
|
|
|
|
{
|
|
|
|
|
var earlyResponse = EmptyResponse(query, request.K, "query_too_long", request.Ambient);
|
|
|
|
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
if (emitObservability)
|
|
|
|
|
{
|
|
|
|
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
return earlyResponse;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -122,7 +207,10 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
if (!_options.Enabled || !IsSearchEnabledForTenant(tenantFlags) || string.IsNullOrWhiteSpace(_options.ConnectionString))
|
|
|
|
|
{
|
|
|
|
|
var earlyResponse = EmptyResponse(query, request.K, "disabled", request.Ambient);
|
|
|
|
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
if (emitObservability)
|
|
|
|
|
{
|
|
|
|
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
return earlyResponse;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -266,7 +354,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
|
|
|
|
|
cards = cards.Take(Math.Max(1, _unifiedOptions.MaxCards)).ToArray();
|
|
|
|
|
|
|
|
|
|
if (!string.IsNullOrWhiteSpace(request.Ambient?.SessionId))
|
|
|
|
|
if (emitObservability && !string.IsNullOrWhiteSpace(request.Ambient?.SessionId))
|
|
|
|
|
{
|
|
|
|
|
_searchSessionContext.RecordQuery(
|
|
|
|
|
tenantId,
|
|
|
|
|
@@ -276,16 +364,21 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
_timeProvider.GetUtcNow());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var currentScopeDomain = ResolveAmbientScopeDomain(request.Ambient);
|
|
|
|
|
var (primaryCards, overflow) = PartitionCardsByScope(cards, currentScopeDomain);
|
|
|
|
|
var visibleCards = BuildVisibleAnswerCards(primaryCards, overflow);
|
|
|
|
|
var coverage = BuildCoverage(currentScopeDomain, merged, primaryCards, overflow);
|
|
|
|
|
|
|
|
|
|
SynthesisResult? synthesis = null;
|
|
|
|
|
if (request.IncludeSynthesis && IsSynthesisEnabledForTenant(tenantFlags) && cards.Count > 0)
|
|
|
|
|
if (request.IncludeSynthesis && IsSynthesisEnabledForTenant(tenantFlags) && visibleCards.Count > 0)
|
|
|
|
|
{
|
|
|
|
|
synthesis = await _synthesisEngine.SynthesizeAsync(
|
|
|
|
|
query, cards, plan.DetectedEntities, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
query, visibleCards, plan.DetectedEntities, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// G4-003: Generate "Did you mean?" suggestions when results are sparse
|
|
|
|
|
IReadOnlyList<SearchSuggestion>? suggestions = null;
|
|
|
|
|
if (cards.Count < _options.MinFtsResultsForFuzzyFallback && _options.FuzzyFallbackEnabled)
|
|
|
|
|
if (visibleCards.Count < _options.MinFtsResultsForFuzzyFallback && _options.FuzzyFallbackEnabled)
|
|
|
|
|
{
|
|
|
|
|
suggestions = await GenerateSuggestionsAsync(
|
|
|
|
|
query, storeFilter, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
@@ -293,10 +386,10 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
|
|
|
|
|
// G10-004: Generate query refinement suggestions from feedback data
|
|
|
|
|
IReadOnlyList<SearchRefinement>? refinements = null;
|
|
|
|
|
if (cards.Count < RefinementResultThreshold)
|
|
|
|
|
if (visibleCards.Count < RefinementResultThreshold)
|
|
|
|
|
{
|
|
|
|
|
refinements = await GenerateRefinementsAsync(
|
|
|
|
|
tenantId, query, cards.Count, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
tenantId, query, visibleCards.Count, storeFilter, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var duration = _timeProvider.GetUtcNow() - startedAt;
|
|
|
|
|
@@ -304,19 +397,22 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
query,
|
|
|
|
|
plan,
|
|
|
|
|
request.Ambient,
|
|
|
|
|
cards,
|
|
|
|
|
primaryCards,
|
|
|
|
|
overflow,
|
|
|
|
|
synthesis,
|
|
|
|
|
suggestions,
|
|
|
|
|
refinements);
|
|
|
|
|
refinements,
|
|
|
|
|
coverage);
|
|
|
|
|
var totalVisibleCardCount = primaryCards.Count + (overflow?.Cards.Count ?? 0);
|
|
|
|
|
var response = new UnifiedSearchResponse(
|
|
|
|
|
query,
|
|
|
|
|
topK,
|
|
|
|
|
cards,
|
|
|
|
|
primaryCards,
|
|
|
|
|
synthesis,
|
|
|
|
|
new UnifiedSearchDiagnostics(
|
|
|
|
|
ftsRows.Count,
|
|
|
|
|
vectorRows.Length,
|
|
|
|
|
cards.Count,
|
|
|
|
|
totalVisibleCardCount,
|
|
|
|
|
(long)duration.TotalMilliseconds,
|
|
|
|
|
usedVector,
|
|
|
|
|
usedVector ? "hybrid" : "fts-only",
|
|
|
|
|
@@ -324,10 +420,16 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
federationDiagnostics),
|
|
|
|
|
suggestions,
|
|
|
|
|
refinements,
|
|
|
|
|
contextAnswer);
|
|
|
|
|
contextAnswer,
|
|
|
|
|
overflow,
|
|
|
|
|
coverage);
|
|
|
|
|
|
|
|
|
|
if (emitObservability)
|
|
|
|
|
{
|
|
|
|
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan, response, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
EmitTelemetry(plan, response, tenantId);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan, response, cancellationToken).ConfigureAwait(false);
|
|
|
|
|
EmitTelemetry(plan, response, tenantId);
|
|
|
|
|
return response;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -388,6 +490,198 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static (IReadOnlyList<EntityCard> PrimaryCards, UnifiedSearchOverflow? Overflow) PartitionCardsByScope(
|
|
|
|
|
IReadOnlyList<EntityCard> rankedCards,
|
|
|
|
|
string? currentScopeDomain)
|
|
|
|
|
{
|
|
|
|
|
if (rankedCards.Count == 0 || string.IsNullOrWhiteSpace(currentScopeDomain))
|
|
|
|
|
{
|
|
|
|
|
return (rankedCards, null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var scopedCards = rankedCards
|
|
|
|
|
.Where(card => string.Equals(card.Domain, currentScopeDomain, StringComparison.OrdinalIgnoreCase))
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
if (scopedCards.Length == 0)
|
|
|
|
|
{
|
|
|
|
|
return (rankedCards, null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var overflowCards = rankedCards
|
|
|
|
|
.Where(card => !string.Equals(card.Domain, currentScopeDomain, StringComparison.OrdinalIgnoreCase))
|
|
|
|
|
.Where(card => ShouldSurfaceOverflow(scopedCards[0].Score, card.Score))
|
|
|
|
|
.Take(MaxOverflowCards)
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
if (overflowCards.Length == 0)
|
|
|
|
|
{
|
|
|
|
|
return (scopedCards, null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return (
|
|
|
|
|
scopedCards,
|
|
|
|
|
new UnifiedSearchOverflow(
|
|
|
|
|
currentScopeDomain,
|
|
|
|
|
BuildOverflowReason(currentScopeDomain, scopedCards[0], overflowCards[0]),
|
|
|
|
|
overflowCards));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static bool ShouldSurfaceOverflow(double topScopedScore, double candidateScore)
|
|
|
|
|
{
|
|
|
|
|
if (candidateScore >= topScopedScore)
|
|
|
|
|
{
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var denominator = Math.Max(Math.Abs(topScopedScore), 0.000001d);
|
|
|
|
|
var relativeGap = (topScopedScore - candidateScore) / denominator;
|
|
|
|
|
return relativeGap <= OverflowScoreBandRatio;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static string BuildOverflowReason(string currentScopeDomain, EntityCard topScopedCard, EntityCard topOverflowCard)
|
|
|
|
|
{
|
|
|
|
|
if (topOverflowCard.Score > topScopedCard.Score)
|
|
|
|
|
{
|
|
|
|
|
return $"Related results outside {DescribeDomain(currentScopeDomain)} outranked the current-scope evidence.";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return $"Related results outside {DescribeDomain(currentScopeDomain)} are close enough in score to surface below the in-scope results.";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static IReadOnlyList<EntityCard> BuildVisibleAnswerCards(
|
|
|
|
|
IReadOnlyList<EntityCard> primaryCards,
|
|
|
|
|
UnifiedSearchOverflow? overflow)
|
|
|
|
|
{
|
|
|
|
|
return primaryCards
|
|
|
|
|
.Concat(overflow?.Cards ?? [])
|
|
|
|
|
.OrderByDescending(static card => card.Score)
|
|
|
|
|
.ThenBy(static card => card.Title, StringComparer.Ordinal)
|
|
|
|
|
.ThenBy(static card => card.EntityKey, StringComparer.Ordinal)
|
|
|
|
|
.ToArray();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static IReadOnlyList<EntityCard> SelectDominantAnswerCards(IReadOnlyList<EntityCard> visibleCards)
|
|
|
|
|
{
|
|
|
|
|
if (visibleCards.Count <= 1)
|
|
|
|
|
{
|
|
|
|
|
return visibleCards.Take(1).ToArray();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var topScore = Math.Max(Math.Abs(visibleCards[0].Score), 0.000001d);
|
|
|
|
|
var blended = visibleCards
|
|
|
|
|
.Where(card => (topScore - card.Score) / topScore <= BlendedAnswerScoreBandRatio)
|
|
|
|
|
.Take(MaxContextAnswerCitations)
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
return blended.Length >= 2
|
|
|
|
|
? blended
|
|
|
|
|
: visibleCards.Take(1).ToArray();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static UnifiedSearchCoverage BuildCoverage(
|
|
|
|
|
string? currentScopeDomain,
|
|
|
|
|
IReadOnlyList<(KnowledgeChunkRow Row, double Score, IReadOnlyDictionary<string, string> Debug)> merged,
|
|
|
|
|
IReadOnlyList<EntityCard> primaryCards,
|
|
|
|
|
UnifiedSearchOverflow? overflow)
|
|
|
|
|
{
|
|
|
|
|
var visibleCounts = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
|
foreach (var card in primaryCards.Concat(overflow?.Cards ?? []))
|
|
|
|
|
{
|
|
|
|
|
visibleCounts[card.Domain] = visibleCounts.TryGetValue(card.Domain, out var existing)
|
|
|
|
|
? existing + 1
|
|
|
|
|
: 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var domains = merged
|
|
|
|
|
.Take(CoverageCandidateWindow)
|
|
|
|
|
.GroupBy(static item => GetDomain(item.Row), StringComparer.OrdinalIgnoreCase)
|
|
|
|
|
.Select(group =>
|
|
|
|
|
{
|
|
|
|
|
var domain = group.Key;
|
|
|
|
|
var visibleCount = visibleCounts.TryGetValue(domain, out var count) ? count : 0;
|
|
|
|
|
return new UnifiedSearchDomainCoverage(
|
|
|
|
|
Domain: domain,
|
|
|
|
|
CandidateCount: group.Count(),
|
|
|
|
|
VisibleCardCount: visibleCount,
|
|
|
|
|
TopScore: group.Max(static item => item.Score),
|
|
|
|
|
IsCurrentScope: string.Equals(domain, currentScopeDomain, StringComparison.OrdinalIgnoreCase),
|
|
|
|
|
HasVisibleResults: visibleCount > 0);
|
|
|
|
|
})
|
|
|
|
|
.ToDictionary(static entry => entry.Domain, StringComparer.OrdinalIgnoreCase);
|
|
|
|
|
|
|
|
|
|
if (!string.IsNullOrWhiteSpace(currentScopeDomain) && !domains.ContainsKey(currentScopeDomain))
|
|
|
|
|
{
|
|
|
|
|
domains[currentScopeDomain] = new UnifiedSearchDomainCoverage(
|
|
|
|
|
Domain: currentScopeDomain,
|
|
|
|
|
CandidateCount: 0,
|
|
|
|
|
VisibleCardCount: 0,
|
|
|
|
|
TopScore: 0d,
|
|
|
|
|
IsCurrentScope: true,
|
|
|
|
|
HasVisibleResults: false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var ordered = domains.Values
|
|
|
|
|
.OrderByDescending(static entry => entry.IsCurrentScope)
|
|
|
|
|
.ThenByDescending(static entry => entry.HasVisibleResults)
|
|
|
|
|
.ThenByDescending(static entry => entry.CandidateCount)
|
|
|
|
|
.ThenByDescending(static entry => entry.TopScore)
|
|
|
|
|
.ThenBy(static entry => entry.Domain, StringComparer.Ordinal)
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
return new UnifiedSearchCoverage(
|
|
|
|
|
CurrentScopeDomain: currentScopeDomain,
|
|
|
|
|
CurrentScopeWeighted: !string.IsNullOrWhiteSpace(currentScopeDomain),
|
|
|
|
|
Domains: ordered);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static UnifiedSearchCoverage MergeCoverage(
|
|
|
|
|
UnifiedSearchCoverage? existing,
|
|
|
|
|
UnifiedSearchCoverage? current)
|
|
|
|
|
{
|
|
|
|
|
if (current is null)
|
|
|
|
|
{
|
|
|
|
|
return existing ?? new UnifiedSearchCoverage(null, false, []);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (existing is null)
|
|
|
|
|
{
|
|
|
|
|
return current;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var domains = new Dictionary<string, UnifiedSearchDomainCoverage>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
|
foreach (var coverage in existing.Domains.Concat(current.Domains))
|
|
|
|
|
{
|
|
|
|
|
if (!domains.TryGetValue(coverage.Domain, out var prior))
|
|
|
|
|
{
|
|
|
|
|
domains[coverage.Domain] = coverage;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
domains[coverage.Domain] = new UnifiedSearchDomainCoverage(
|
|
|
|
|
Domain: coverage.Domain,
|
|
|
|
|
CandidateCount: Math.Max(prior.CandidateCount, coverage.CandidateCount),
|
|
|
|
|
VisibleCardCount: Math.Max(prior.VisibleCardCount, coverage.VisibleCardCount),
|
|
|
|
|
TopScore: Math.Max(prior.TopScore, coverage.TopScore),
|
|
|
|
|
IsCurrentScope: prior.IsCurrentScope || coverage.IsCurrentScope,
|
|
|
|
|
HasVisibleResults: prior.HasVisibleResults || coverage.HasVisibleResults);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var currentScopeDomain = current.CurrentScopeDomain
|
|
|
|
|
?? existing.CurrentScopeDomain;
|
|
|
|
|
|
|
|
|
|
return new UnifiedSearchCoverage(
|
|
|
|
|
CurrentScopeDomain: currentScopeDomain,
|
|
|
|
|
CurrentScopeWeighted: existing.CurrentScopeWeighted || current.CurrentScopeWeighted,
|
|
|
|
|
Domains: domains.Values
|
|
|
|
|
.OrderByDescending(static entry => entry.IsCurrentScope)
|
|
|
|
|
.ThenByDescending(static entry => entry.HasVisibleResults)
|
|
|
|
|
.ThenByDescending(static entry => entry.CandidateCount)
|
|
|
|
|
.ThenByDescending(static entry => entry.TopScore)
|
|
|
|
|
.ThenBy(static entry => entry.Domain, StringComparer.Ordinal)
|
|
|
|
|
.ToArray());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static IReadOnlyList<ContextAnswerCitation> BuildContextAnswerCitations(
|
|
|
|
|
IReadOnlyList<EntityCard> cards,
|
|
|
|
|
SynthesisResult? synthesis)
|
|
|
|
|
@@ -441,7 +735,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
return citations;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static string BuildGroundedSummary(IReadOnlyList<EntityCard> cards, SynthesisResult? synthesis)
|
|
|
|
|
private static string BuildGroundedSummary(IReadOnlyList<EntityCard> answerCards, SynthesisResult? synthesis)
|
|
|
|
|
{
|
|
|
|
|
var synthesisSummary = synthesis?.Summary?.Trim();
|
|
|
|
|
if (!string.IsNullOrWhiteSpace(synthesisSummary))
|
|
|
|
|
@@ -449,27 +743,71 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
return synthesisSummary;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var topCard = cards[0];
|
|
|
|
|
var topCard = answerCards[0];
|
|
|
|
|
if (answerCards.Count > 1)
|
|
|
|
|
{
|
|
|
|
|
var related = answerCards
|
|
|
|
|
.Skip(1)
|
|
|
|
|
.Select(static card => card.Title)
|
|
|
|
|
.Where(static title => !string.IsNullOrWhiteSpace(title))
|
|
|
|
|
.Take(2)
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
if (related.Length > 0)
|
|
|
|
|
{
|
|
|
|
|
return $"Top evidence points to {topCard.Title}. Related high-confidence matches also include {string.Join(", ", related)}.";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var snippet = topCard.Snippet?.Trim();
|
|
|
|
|
return string.IsNullOrWhiteSpace(snippet) ? topCard.Title : snippet;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static string BuildGroundedReason(QueryPlan plan, AmbientContext? ambient, EntityCard topCard)
|
|
|
|
|
private static string BuildGroundedReason(
|
|
|
|
|
QueryPlan plan,
|
|
|
|
|
AmbientContext? ambient,
|
|
|
|
|
IReadOnlyList<EntityCard> answerCards,
|
|
|
|
|
UnifiedSearchCoverage? coverage,
|
|
|
|
|
UnifiedSearchOverflow? overflow)
|
|
|
|
|
{
|
|
|
|
|
var scope = ResolveContextDomain(plan, [topCard], ambient) ?? topCard.Domain;
|
|
|
|
|
var topCard = answerCards[0];
|
|
|
|
|
var scope = coverage?.CurrentScopeDomain
|
|
|
|
|
?? ResolveContextDomain(plan, [topCard], ambient)
|
|
|
|
|
?? topCard.Domain;
|
|
|
|
|
|
|
|
|
|
if (answerCards.Count > 1)
|
|
|
|
|
{
|
|
|
|
|
return $"The highest-ranked results are close in score, so the answer blends evidence across {FormatDomainList(answerCards.Select(static card => card.Domain))}.";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (overflow is not null)
|
|
|
|
|
{
|
|
|
|
|
return $"Current-scope weighting kept {DescribeDomain(scope)} first, while close related evidence from other domains remains visible below.";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return $"The top result is grounded in {DescribeDomain(scope)} evidence and aligns with the {plan.Intent} intent.";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static string BuildGroundedEvidence(IReadOnlyList<EntityCard> cards, SynthesisResult? synthesis)
|
|
|
|
|
private static string BuildGroundedEvidence(
|
|
|
|
|
IReadOnlyList<EntityCard> visibleCards,
|
|
|
|
|
IReadOnlyList<EntityCard> answerCards,
|
|
|
|
|
SynthesisResult? synthesis,
|
|
|
|
|
UnifiedSearchCoverage? coverage,
|
|
|
|
|
UnifiedSearchOverflow? overflow)
|
|
|
|
|
{
|
|
|
|
|
var sourceCount = Math.Max(synthesis?.SourceCount ?? 0, cards.Count);
|
|
|
|
|
var sourceCount = Math.Max(synthesis?.SourceCount ?? 0, visibleCards.Count);
|
|
|
|
|
var domains = synthesis?.DomainsCovered is { Count: > 0 }
|
|
|
|
|
? synthesis.DomainsCovered
|
|
|
|
|
: cards.Select(static card => card.Domain)
|
|
|
|
|
: answerCards.Select(static card => card.Domain)
|
|
|
|
|
.Where(static domain => !string.IsNullOrWhiteSpace(domain))
|
|
|
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
|
|
|
.ToArray();
|
|
|
|
|
|
|
|
|
|
if (overflow is not null && coverage?.CurrentScopeDomain is { Length: > 0 } currentScopeDomain)
|
|
|
|
|
{
|
|
|
|
|
return $"Grounded in {sourceCount} source(s) across {FormatDomainList(domains)} with {DescribeDomain(currentScopeDomain)} weighted first.";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return $"Grounded in {sourceCount} source(s) across {FormatDomainList(domains)}.";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -826,6 +1164,28 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
.FirstOrDefault(static value => !string.IsNullOrWhiteSpace(value));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static string? ResolveAmbientScopeDomain(AmbientContext? ambient)
|
|
|
|
|
{
|
|
|
|
|
var routeDomain = AmbientContextProcessor.ResolveDomainFromRoute(ambient?.CurrentRoute ?? string.Empty);
|
|
|
|
|
if (!string.IsNullOrWhiteSpace(routeDomain))
|
|
|
|
|
{
|
|
|
|
|
return routeDomain;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!string.IsNullOrWhiteSpace(ambient?.LastAction?.Domain))
|
|
|
|
|
{
|
|
|
|
|
return ambient.LastAction.Domain.Trim().ToLowerInvariant();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var actionRouteDomain = AmbientContextProcessor.ResolveDomainFromRoute(ambient?.LastAction?.Route ?? string.Empty);
|
|
|
|
|
if (!string.IsNullOrWhiteSpace(actionRouteDomain))
|
|
|
|
|
{
|
|
|
|
|
return actionRouteDomain;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static string DescribeDomain(string domain)
|
|
|
|
|
{
|
|
|
|
|
return domain switch
|
|
|
|
|
@@ -1406,26 +1766,34 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
QueryPlan plan,
|
|
|
|
|
AmbientContext? ambient,
|
|
|
|
|
IReadOnlyList<EntityCard> cards,
|
|
|
|
|
UnifiedSearchOverflow? overflow,
|
|
|
|
|
SynthesisResult? synthesis,
|
|
|
|
|
IReadOnlyList<SearchSuggestion>? suggestions,
|
|
|
|
|
IReadOnlyList<SearchRefinement>? refinements)
|
|
|
|
|
IReadOnlyList<SearchRefinement>? refinements,
|
|
|
|
|
UnifiedSearchCoverage? coverage)
|
|
|
|
|
{
|
|
|
|
|
if (string.IsNullOrWhiteSpace(query))
|
|
|
|
|
{
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (cards.Count > 0)
|
|
|
|
|
var visibleCards = BuildVisibleAnswerCards(cards, overflow);
|
|
|
|
|
if (visibleCards.Count > 0)
|
|
|
|
|
{
|
|
|
|
|
var topCard = cards[0];
|
|
|
|
|
var citations = BuildContextAnswerCitations(cards, synthesis);
|
|
|
|
|
var answerCards = SelectDominantAnswerCards(visibleCards);
|
|
|
|
|
var topCard = answerCards[0];
|
|
|
|
|
var citations = BuildContextAnswerCitations(answerCards, synthesis);
|
|
|
|
|
var questions = BuildGroundedQuestions(query, plan, ambient, topCard);
|
|
|
|
|
return new ContextAnswer(
|
|
|
|
|
Status: "grounded",
|
|
|
|
|
Code: "retrieved_evidence",
|
|
|
|
|
Summary: BuildGroundedSummary(cards, synthesis),
|
|
|
|
|
Reason: BuildGroundedReason(plan, ambient, topCard),
|
|
|
|
|
Evidence: BuildGroundedEvidence(cards, synthesis),
|
|
|
|
|
Code: answerCards.Count > 1
|
|
|
|
|
? "retrieved_blended_evidence"
|
|
|
|
|
: overflow is not null
|
|
|
|
|
? "retrieved_scope_weighted_evidence"
|
|
|
|
|
: "retrieved_evidence",
|
|
|
|
|
Summary: BuildGroundedSummary(answerCards, synthesis),
|
|
|
|
|
Reason: BuildGroundedReason(plan, ambient, answerCards, coverage, overflow),
|
|
|
|
|
Evidence: BuildGroundedEvidence(visibleCards, answerCards, synthesis, coverage, overflow),
|
|
|
|
|
Citations: citations,
|
|
|
|
|
Questions: questions);
|
|
|
|
|
}
|
|
|
|
|
@@ -1581,7 +1949,11 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
suggestions.Add(new SearchSuggestion(text, $"Similar to \"{query}\""));
|
|
|
|
|
suggestions.Add(new SearchSuggestion(
|
|
|
|
|
Text: text,
|
|
|
|
|
Reason: $"Similar to \"{query}\"",
|
|
|
|
|
Domain: GetDomain(row),
|
|
|
|
|
CandidateCount: 1));
|
|
|
|
|
|
|
|
|
|
if (suggestions.Count >= maxSuggestions)
|
|
|
|
|
{
|
|
|
|
|
@@ -1672,7 +2044,11 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
/// Sprint: G10-004
|
|
|
|
|
/// </summary>
|
|
|
|
|
private async Task<IReadOnlyList<SearchRefinement>?> GenerateRefinementsAsync(
|
|
|
|
|
string tenantId, string query, int resultCount, CancellationToken ct)
|
|
|
|
|
string tenantId,
|
|
|
|
|
string query,
|
|
|
|
|
int resultCount,
|
|
|
|
|
KnowledgeSearchFilter? storeFilter,
|
|
|
|
|
CancellationToken ct)
|
|
|
|
|
{
|
|
|
|
|
if (resultCount >= RefinementResultThreshold)
|
|
|
|
|
{
|
|
|
|
|
@@ -1705,9 +2081,14 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
var text = alert.Resolution.Trim();
|
|
|
|
|
if (text.Length > 120) text = text[..120].TrimEnd();
|
|
|
|
|
|
|
|
|
|
if (seen.Add(text))
|
|
|
|
|
var probe = await ProbeCandidateAsync(text, storeFilter, refinementCt).ConfigureAwait(false);
|
|
|
|
|
if (probe is not null && seen.Add(text))
|
|
|
|
|
{
|
|
|
|
|
refinements.Add(new SearchRefinement(text, "resolved_alert"));
|
|
|
|
|
refinements.Add(new SearchRefinement(
|
|
|
|
|
Text: text,
|
|
|
|
|
Source: "resolved_alert",
|
|
|
|
|
Domain: probe.Value.Domain,
|
|
|
|
|
CandidateCount: probe.Value.CandidateCount));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -1721,9 +2102,14 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
{
|
|
|
|
|
if (refinements.Count >= maxRefinements) break;
|
|
|
|
|
|
|
|
|
|
if (seen.Add(similarQuery))
|
|
|
|
|
var probe = await ProbeCandidateAsync(similarQuery, storeFilter, refinementCt).ConfigureAwait(false);
|
|
|
|
|
if (probe is not null && seen.Add(similarQuery))
|
|
|
|
|
{
|
|
|
|
|
refinements.Add(new SearchRefinement(similarQuery, "similar_successful_query"));
|
|
|
|
|
refinements.Add(new SearchRefinement(
|
|
|
|
|
Text: similarQuery,
|
|
|
|
|
Source: "similar_successful_query",
|
|
|
|
|
Domain: probe.Value.Domain,
|
|
|
|
|
CandidateCount: probe.Value.CandidateCount));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@@ -1737,9 +2123,19 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
{
|
|
|
|
|
if (refinements.Count >= maxRefinements) break;
|
|
|
|
|
|
|
|
|
|
if (!string.IsNullOrWhiteSpace(entityKey) && seen.Add(entityKey))
|
|
|
|
|
if (string.IsNullOrWhiteSpace(entityKey))
|
|
|
|
|
{
|
|
|
|
|
refinements.Add(new SearchRefinement(entityKey, "entity_alias"));
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var probe = await ProbeCandidateAsync(entityKey, storeFilter, refinementCt).ConfigureAwait(false);
|
|
|
|
|
if (probe is not null && seen.Add(entityKey))
|
|
|
|
|
{
|
|
|
|
|
refinements.Add(new SearchRefinement(
|
|
|
|
|
Text: entityKey,
|
|
|
|
|
Source: "entity_alias",
|
|
|
|
|
Domain: probe.Value.Domain,
|
|
|
|
|
CandidateCount: probe.Value.CandidateCount));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@@ -1759,6 +2155,42 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|
|
|
|
return refinements.Count > 0 ? refinements : null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async Task<(string Domain, int CandidateCount)?> ProbeCandidateAsync(
|
|
|
|
|
string candidateQuery,
|
|
|
|
|
KnowledgeSearchFilter? storeFilter,
|
|
|
|
|
CancellationToken cancellationToken)
|
|
|
|
|
{
|
|
|
|
|
var normalized = KnowledgeSearchText.NormalizeWhitespace(candidateQuery);
|
|
|
|
|
if (string.IsNullOrWhiteSpace(normalized))
|
|
|
|
|
{
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var timeout = TimeSpan.FromMilliseconds(Math.Clamp(_options.QueryTimeoutMs / 6, 50, 500));
|
|
|
|
|
var rows = await _store.SearchFtsAsync(
|
|
|
|
|
normalized,
|
|
|
|
|
storeFilter,
|
|
|
|
|
3,
|
|
|
|
|
timeout,
|
|
|
|
|
cancellationToken).ConfigureAwait(false);
|
|
|
|
|
|
|
|
|
|
if (rows.Count == 0)
|
|
|
|
|
{
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var dominantDomain = rows
|
|
|
|
|
.GroupBy(GetDomain, StringComparer.OrdinalIgnoreCase)
|
|
|
|
|
.OrderByDescending(static group => group.Count())
|
|
|
|
|
.ThenBy(static group => group.Key, StringComparer.Ordinal)
|
|
|
|
|
.Select(static group => group.Key)
|
|
|
|
|
.FirstOrDefault();
|
|
|
|
|
|
|
|
|
|
return dominantDomain is null
|
|
|
|
|
? null
|
|
|
|
|
: (dominantDomain, rows.Count);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
|
|
/// Computes Jaccard similarity over character trigrams of two strings.
|
|
|
|
|
/// Used as an in-memory approximation of PostgreSQL pg_trgm similarity().
|
|
|
|
|
|