Add self-serve search telemetry gap surfacing

This commit is contained in:
master
2026-03-07 17:15:38 +02:00
parent 5fac47f99f
commit 14d7612cc2
13 changed files with 677 additions and 28 deletions

View File

@@ -65,7 +65,7 @@ Completion criteria:
- [ ] Ambiguous queries return clarifying prompts instead of a blank answer slot.
### AI-SELF-004 - Self-serve telemetry and gap surfacing
Status: TODO
Status: DONE
Dependency: AI-SELF-002
Owners: Developer (AdvisoryAI), Test Automation
Task description:
@@ -73,9 +73,9 @@ Task description:
- Expose enough structured data to drive a gap-closure backlog.
Completion criteria:
- [ ] Telemetry captures unanswered and reformulated journeys without persisting raw sensitive prompts unnecessarily.
- [ ] Operational docs explain how to review self-serve gaps.
- [ ] Tests cover telemetry emission for fallback paths.
- [x] Telemetry captures unanswered and reformulated journeys without persisting raw sensitive prompts unnecessarily.
- [x] Operational docs explain how to review self-serve gaps.
- [x] Tests cover telemetry emission for fallback paths.
### AI-SELF-005 - Targeted behavioral verification
Status: DONE
@@ -110,6 +110,8 @@ Completion criteria:
| 2026-03-07 | Implemented `contextAnswer` in unified search/backend API mapping, added deterministic `grounded` / `clarify` / `insufficient` rules plus follow-up question generation, and extended telemetry fields for answer-state visibility. | Developer |
| 2026-03-07 | Verified the AdvisoryAI test project after the contract change with `dotnet test "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj" --no-restore -v normal` (`877/877` passing). | Test Automation |
| 2026-03-07 | Exercised the live rebuilt-corpus lane against `http://127.0.0.1:10451`: `POST /v1/advisory-ai/index/rebuild`, `POST /v1/search/index/rebuild`, then `POST /v1/search/query` for `database connectivity`, which returned `contextAnswer.status = grounded`, 3 citations, and 10 cards over ingested data. | Test Automation |
| 2026-03-07 | Implemented optional self-serve analytics columns (`session_id`, `answer_status`, `answer_code`), metrics (`fallbackAnswerRate`, `clarifyRate`, `insufficientRate`, `reformulationCount`, `rescueActionCount`, `abandonedFallbackCount`), and alerting (`fallback_loop`, `abandoned_fallback`) with privacy-preserving session hashing. | Developer |
| 2026-03-07 | Verified targeted telemetry coverage with xUnit v3 runner commands against `UnifiedSearchSprintIntegrationTests` and added a recovered-session regression test so rescue actions clear abandonment as expected. | Test Automation |
## Decisions & Risks
- Decision: the backend contract must return explicit answer states instead of leaving the UI to infer confidence from cards alone.
@@ -122,6 +124,8 @@ Completion criteria:
- Risk: mocked endpoint tests can overstate confidence if ingestion adapters or corpus rebuild order drift.
- Mitigation: keep rebuild order documented, execute it during verification, and record which routes have live-ingested parity.
- Decision: `stella advisoryai sources prepare` is optional for local verification when checked-in Doctor seed/control files are already sufficient, but it requires `STELLAOPS_BACKEND_URL` whenever live Doctor discovery is expected.
- Decision: self-serve analytics is optional and additive; search behavior must not depend on telemetry event emission.
- Decision: targeted AdvisoryAI verification uses xUnit v3 / Microsoft.Testing.Platform-compatible filters (`-- --filter-class` or the built test executable), not legacy VSTest `--filter`.
## Next Checkpoints
- 2026-03-10: Freeze answer payload shape and fallback taxonomy.

View File

@@ -17,7 +17,10 @@ public static class SearchAnalyticsEndpoints
"query",
"click",
"zero_result",
"synthesis"
"synthesis",
"answer_frame",
"reformulation",
"rescue_action"
};
public static RouteGroupBuilder MapSearchAnalyticsEndpoints(this IEndpointRouteBuilder builder)
@@ -30,10 +33,11 @@ public static class SearchAnalyticsEndpoints
group.MapPost("/analytics", RecordAnalyticsAsync)
.WithName("SearchAnalyticsRecord")
.WithSummary("Records batch search analytics events (query, click, zero_result, synthesis).")
.WithSummary("Records batch search analytics events (query, click, zero_result, synthesis, answer_frame, reformulation, rescue_action).")
.WithDescription(
"Accepts a batch of search analytics events for tracking query frequency, click-through rates, " +
"zero-result queries, and synthesis usage. Queries and user identifiers are pseudonymized before persistence. " +
"zero-result queries, self-serve answer states, reformulations, rescue-action usage, and synthesis usage. " +
"Queries and user/session identifiers are pseudonymized before persistence. " +
"Fire-and-forget from the client; failures do not affect search functionality.")
.RequireAuthorization(AdvisoryAIPolicies.OperatePolicy)
.Produces(StatusCodes.Status204NoContent)
@@ -113,11 +117,14 @@ public static class SearchAnalyticsEndpoints
EventType: apiEvent.EventType.Trim().ToLowerInvariant(),
Query: apiEvent.Query.Trim(),
UserId: userId,
SessionId: string.IsNullOrWhiteSpace(apiEvent.SessionId) ? null : apiEvent.SessionId.Trim(),
EntityKey: string.IsNullOrWhiteSpace(apiEvent.EntityKey) ? null : apiEvent.EntityKey.Trim(),
Domain: string.IsNullOrWhiteSpace(apiEvent.Domain) ? null : apiEvent.Domain.Trim(),
ResultCount: apiEvent.ResultCount,
Position: apiEvent.Position,
DurationMs: apiEvent.DurationMs));
DurationMs: apiEvent.DurationMs,
AnswerStatus: string.IsNullOrWhiteSpace(apiEvent.AnswerStatus) ? null : apiEvent.AnswerStatus.Trim(),
AnswerCode: string.IsNullOrWhiteSpace(apiEvent.AnswerCode) ? null : apiEvent.AnswerCode.Trim()));
}
if (events.Count > 0)
@@ -301,6 +308,8 @@ public sealed record SearchAnalyticsApiEvent
public string Query { get; init; } = string.Empty;
public string? SessionId { get; init; }
public string? EntityKey { get; init; }
public string? Domain { get; init; }
@@ -310,6 +319,10 @@ public sealed record SearchAnalyticsApiEvent
public int? Position { get; init; }
public int? DurationMs { get; init; }
public string? AnswerStatus { get; init; }
public string? AnswerCode { get; init; }
}
public sealed record SearchHistoryApiResponse

View File

@@ -39,11 +39,11 @@ public static class SearchFeedbackEndpoints
// G10-002: List quality alerts (admin only)
group.MapGet("/quality/alerts", GetAlertsAsync)
.WithName("SearchQualityAlertsList")
.WithSummary("Lists open search quality alerts (zero-result queries, high negative feedback).")
.WithSummary("Lists open search quality alerts (zero-result, high negative feedback, fallback loops, abandoned fallback).")
.WithDescription(
"Returns search quality alerts ordered by occurrence count. " +
"Filterable by status (open, acknowledged, resolved) and alert type " +
"(zero_result, low_feedback, high_negative_feedback). Requires admin scope.")
"(zero_result, high_negative_feedback, fallback_loop, abandoned_fallback). Requires admin scope.")
.RequireAuthorization(AdvisoryAIPolicies.AdminPolicy)
.Produces<IReadOnlyList<SearchQualityAlertDto>>(StatusCodes.Status200OK)
.Produces(StatusCodes.Status403Forbidden);
@@ -198,6 +198,12 @@ public static class SearchFeedbackEndpoints
ZeroResultRate = metrics.ZeroResultRate,
AvgResultCount = metrics.AvgResultCount,
FeedbackScore = metrics.FeedbackScore,
FallbackAnswerRate = metrics.FallbackAnswerRate,
ClarifyRate = metrics.ClarifyRate,
InsufficientRate = metrics.InsufficientRate,
ReformulationCount = metrics.ReformulationCount,
RescueActionCount = metrics.RescueActionCount,
AbandonedFallbackCount = metrics.AbandonedFallbackCount,
Period = metrics.Period,
LowQualityResults = metrics.LowQualityResults
.Select(row => new SearchLowQualityResultDto
@@ -308,6 +314,12 @@ public sealed record SearchQualityMetricsDto
public double ZeroResultRate { get; init; }
public double AvgResultCount { get; init; }
public double FeedbackScore { get; init; }
public double FallbackAnswerRate { get; init; }
public double ClarifyRate { get; init; }
public double InsufficientRate { get; init; }
public int ReformulationCount { get; init; }
public int RescueActionCount { get; init; }
public int AbandonedFallbackCount { get; init; }
public string Period { get; init; } = "7d";
public IReadOnlyList<SearchLowQualityResultDto> LowQualityResults { get; init; } = [];
public IReadOnlyList<SearchTopQueryDto> TopQueries { get; init; } = [];

View File

@@ -17,4 +17,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| SPRINT_20260224_G1-G10 | DONE | Search improvement sprints G1G10 implemented. New endpoints: `SearchAnalyticsEndpoints.cs` (history, events, popularity), `SearchFeedbackEndpoints.cs` (feedback, quality alerts, metrics). Extended: `UnifiedSearchEndpoints.cs` (suggestions, refinements, previews, diagnostics.activeEncoder). Extended: `KnowledgeSearchEndpoints.cs` (activeEncoder in diagnostics). See `docs/modules/advisory-ai/knowledge-search.md` for full testing guide. |
| AI-SELF-001 | DONE | Unified search endpoint contract now exposes backend contextual answer fields for self-serve search. |
| AI-SELF-004 | DONE | Search analytics and quality endpoints now surface self-serve metrics and alerts (`fallback_loop`, `abandoned_fallback`) while keeping telemetry optional. |
| AI-SELF-006 | DONE | Endpoint readiness now includes a proven local rebuilt-corpus verification lane in addition to stubbed integration tests. |

View File

@@ -0,0 +1,20 @@
-- 008_search_self_serve_analytics.sql
-- Adds self-serve telemetry fields to search_events so fallback loops,
-- reformulations, rescue actions, and abandoned fallback sessions can be tracked.
ALTER TABLE advisoryai.search_events
ADD COLUMN IF NOT EXISTS session_id TEXT;
ALTER TABLE advisoryai.search_events
ADD COLUMN IF NOT EXISTS answer_status TEXT;
ALTER TABLE advisoryai.search_events
ADD COLUMN IF NOT EXISTS answer_code TEXT;
CREATE INDEX IF NOT EXISTS idx_search_events_tenant_session
ON advisoryai.search_events (tenant_id, session_id, created_at)
WHERE session_id IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_search_events_answer_status
ON advisoryai.search_events (tenant_id, answer_status, created_at)
WHERE answer_status IS NOT NULL;

View File

@@ -13,7 +13,7 @@ Source of truth: `docs/implplan/SPRINT_20260113_005_ADVISORYAI_controlled_conver
| AI-SELF-001 | DONE | Unified search now emits the contextual answer payload (`contextAnswer`) with answer state, citations, and follow-up questions for self-serve search. |
| AI-SELF-002 | DONE | Deterministic grounded/clarify/insufficient fallback policy is implemented in unified search orchestration. |
| AI-SELF-003 | DONE | Follow-up question generation from route/domain intent, recent actions, and evidence is implemented. |
| AI-SELF-004 | TODO | Telemetry for unanswered and reformulated self-serve journeys is still pending. |
| AI-SELF-004 | DONE | Optional self-serve telemetry now captures answer frames, reformulations, rescue actions, and abandonment signals via hashed session ids; quality metrics and alerts expose the gaps operationally. |
| AI-SELF-006 | DONE | Live ingestion-backed answer verification succeeded on the Doctor/knowledge route after local rebuild. |
| SPRINT_20260222_051-AKS-INGEST | DONE | Added deterministic AKS ingestion controls: markdown allow-list manifest loading, OpenAPI aggregate source path support, and doctor control projection integration for search chunks, including fallback doctor metadata hydration from controls projection fields. |
| AUDIT-0017-M | DONE | Maintainability audit for StellaOps.AdvisoryAI. |

View File

@@ -32,6 +32,19 @@ internal static class SearchAnalyticsPrivacy
return Convert.ToHexString(hash).ToLowerInvariant();
}
public static string? HashSessionId(string tenantId, string? sessionId)
{
if (string.IsNullOrWhiteSpace(sessionId))
{
return null;
}
var normalizedTenant = tenantId.Trim().ToLowerInvariant();
var normalizedSession = sessionId.Trim().ToLowerInvariant();
var hash = SHA256.HashData(Encoding.UTF8.GetBytes($"{normalizedTenant}|session|{normalizedSession}"));
return Convert.ToHexString(hash).ToLowerInvariant();
}
public static string? RedactFreeform(string? value)
{
_ = value;

View File

@@ -37,11 +37,14 @@ internal sealed class SearchAnalyticsService
await conn.OpenAsync(ct).ConfigureAwait(false);
await using var cmd = new NpgsqlCommand(@"
INSERT INTO advisoryai.search_events (tenant_id, user_id, event_type, query, entity_key, domain, result_count, position, duration_ms)
VALUES (@tenant_id, @user_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms)", conn);
INSERT INTO advisoryai.search_events
(tenant_id, user_id, session_id, event_type, query, entity_key, domain, result_count, position, duration_ms, answer_status, answer_code)
VALUES
(@tenant_id, @user_id, @session_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms, @answer_status, @answer_code)", conn);
cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId);
cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value);
cmd.Parameters.AddWithValue("session_id", (object?)persistedEvent.SessionId ?? DBNull.Value);
cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType);
cmd.Parameters.AddWithValue("query", persistedEvent.Query);
cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value);
@@ -49,6 +52,8 @@ internal sealed class SearchAnalyticsService
cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value);
cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value);
cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value);
cmd.Parameters.AddWithValue("answer_status", (object?)persistedEvent.AnswerStatus ?? DBNull.Value);
cmd.Parameters.AddWithValue("answer_code", (object?)persistedEvent.AnswerCode ?? DBNull.Value);
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
RecordFallbackEvent(persistedEvent, recordedAt);
@@ -88,11 +93,14 @@ internal sealed class SearchAnalyticsService
var persistedEvent = SanitizeEvent(evt);
await using var cmd = new NpgsqlCommand(@"
INSERT INTO advisoryai.search_events (tenant_id, user_id, event_type, query, entity_key, domain, result_count, position, duration_ms)
VALUES (@tenant_id, @user_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms)", conn);
INSERT INTO advisoryai.search_events
(tenant_id, user_id, session_id, event_type, query, entity_key, domain, result_count, position, duration_ms, answer_status, answer_code)
VALUES
(@tenant_id, @user_id, @session_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms, @answer_status, @answer_code)", conn);
cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId);
cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value);
cmd.Parameters.AddWithValue("session_id", (object?)persistedEvent.SessionId ?? DBNull.Value);
cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType);
cmd.Parameters.AddWithValue("query", persistedEvent.Query);
cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value);
@@ -100,6 +108,8 @@ internal sealed class SearchAnalyticsService
cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value);
cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value);
cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value);
cmd.Parameters.AddWithValue("answer_status", (object?)persistedEvent.AnswerStatus ?? DBNull.Value);
cmd.Parameters.AddWithValue("answer_code", (object?)persistedEvent.AnswerCode ?? DBNull.Value);
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
RecordFallbackEvent(persistedEvent, recordedAt);
@@ -637,8 +647,16 @@ internal sealed class SearchAnalyticsService
{
return evt with
{
EventType = evt.EventType.Trim().ToLowerInvariant(),
Query = SearchAnalyticsPrivacy.HashQuery(evt.Query),
UserId = SearchAnalyticsPrivacy.HashUserId(evt.TenantId, evt.UserId)
UserId = SearchAnalyticsPrivacy.HashUserId(evt.TenantId, evt.UserId),
SessionId = SearchAnalyticsPrivacy.HashSessionId(evt.TenantId, evt.SessionId),
AnswerStatus = string.IsNullOrWhiteSpace(evt.AnswerStatus)
? null
: evt.AnswerStatus.Trim().ToLowerInvariant(),
AnswerCode = string.IsNullOrWhiteSpace(evt.AnswerCode)
? null
: evt.AnswerCode.Trim().ToLowerInvariant()
};
}
@@ -667,11 +685,14 @@ internal record SearchAnalyticsEvent(
string EventType,
string Query,
string? UserId = null,
string? SessionId = null,
string? EntityKey = null,
string? Domain = null,
int? ResultCount = null,
int? Position = null,
int? DurationMs = null);
int? DurationMs = null,
string? AnswerStatus = null,
string? AnswerCode = null);
internal record SearchHistoryEntry(
string HistoryId,

View File

@@ -1,3 +1,4 @@
using System.Globalization;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Npgsql;
@@ -6,7 +7,8 @@ using StellaOps.AdvisoryAI.KnowledgeSearch;
namespace StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
/// <summary>
/// Monitors search quality by analysing feedback data and zero-result queries.
/// Monitors search quality by analysing feedback data, zero-result queries,
/// and self-serve fallback telemetry.
/// Provides CRUD for search_quality_alerts and search_feedback tables.
/// Sprint: SPRINT_20260224_110 (G10-001, G10-002)
/// </summary>
@@ -17,6 +19,8 @@ internal sealed class SearchQualityMonitor
private const int DefaultAlertWindowDays = 7;
private const int ZeroResultAlertThreshold = 3;
private const int NegativeFeedbackAlertThreshold = 3;
private const int FallbackLoopAlertThreshold = 3;
private const int AbandonedFallbackAlertThreshold = 3;
private readonly KnowledgeSearchOptions _options;
private readonly ILogger<SearchQualityMonitor> _logger;
@@ -175,6 +179,32 @@ internal sealed class SearchQualityMonitor
candidate.LastSeen,
ct).ConfigureAwait(false);
}
var selfServeSignals = await LoadSelfServeSignalEventsAsync(tenantId, window, ct).ConfigureAwait(false);
foreach (var candidate in BuildFallbackLoopCandidates(selfServeSignals))
{
await UpsertAlertAsync(
tenantId,
alertType: "fallback_loop",
candidate.Query,
candidate.OccurrenceCount,
candidate.FirstSeen,
candidate.LastSeen,
ct).ConfigureAwait(false);
}
foreach (var candidate in BuildAbandonedFallbackCandidates(selfServeSignals))
{
await UpsertAlertAsync(
tenantId,
alertType: "abandoned_fallback",
candidate.Query,
candidate.OccurrenceCount,
candidate.FirstSeen,
candidate.LastSeen,
ct).ConfigureAwait(false);
}
}
public async Task<IReadOnlyList<SearchQualityAlertEntry>> GetAlertsAsync(
@@ -423,6 +453,9 @@ internal sealed class SearchQualityMonitor
}
await feedbackReader.CloseAsync().ConfigureAwait(false);
var selfServeSignals = await LoadSelfServeSignalEventsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
ApplySelfServeMetrics(metrics, selfServeSignals);
metrics.LowQualityResults = await LoadLowQualityResultsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
metrics.TopQueries = await LoadTopQueriesAsync(conn, tenantId, days, ct).ConfigureAwait(false);
metrics.Trend = await LoadTrendPointsAsync(conn, tenantId, ct).ConfigureAwait(false);
@@ -473,7 +506,8 @@ internal sealed class SearchQualityMonitor
? 0d
: (double)helpfulCount / feedbackSignals.Length * 100d;
return new SearchQualityMetricsEntry
var selfServeSignals = BuildFallbackSelfServeSignalEvents(tenantId, window);
var metrics = new SearchQualityMetricsEntry
{
TotalSearches = totalSearches,
ZeroResultRate = totalSearches == 0 ? 0d : Math.Round((double)zeroResults / totalSearches * 100d, 1),
@@ -484,6 +518,10 @@ internal sealed class SearchQualityMonitor
TopQueries = BuildFallbackTopQueries(tenantId, window),
Trend = BuildFallbackTrendPoints(tenantId),
};
ApplySelfServeMetrics(metrics, selfServeSignals);
return metrics;
}
private async Task<IReadOnlyList<SearchQualityLowQualityRow>> LoadLowQualityResultsAsync(
@@ -758,6 +796,232 @@ internal sealed class SearchQualityMonitor
return points;
}
private async Task<IReadOnlyList<SelfServeSignalEvent>> LoadSelfServeSignalEventsAsync(
string tenantId,
TimeSpan window,
CancellationToken ct)
{
var days = Math.Max(1, (int)Math.Ceiling(window.TotalDays));
if (!string.IsNullOrWhiteSpace(_options.ConnectionString))
{
try
{
await using var conn = new NpgsqlConnection(_options.ConnectionString);
await conn.OpenAsync(ct).ConfigureAwait(false);
return await LoadSelfServeSignalEventsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Failed to load self-serve signal events from database.");
}
}
return BuildFallbackSelfServeSignalEvents(tenantId, window);
}
private static async Task<IReadOnlyList<SelfServeSignalEvent>> LoadSelfServeSignalEventsAsync(
NpgsqlConnection conn,
string tenantId,
int days,
CancellationToken ct)
{
var events = new List<SelfServeSignalEvent>();
await using var cmd = new NpgsqlCommand(@"
SELECT
COALESCE(session_id, ''),
event_type,
query,
answer_status,
created_at,
event_id::text
FROM advisoryai.search_events
WHERE tenant_id = @tenant_id
AND created_at > now() - make_interval(days => @days)
AND event_type IN ('answer_frame', 'reformulation', 'rescue_action', 'click')
ORDER BY created_at ASC, event_id ASC", conn);
cmd.Parameters.AddWithValue("tenant_id", tenantId);
cmd.Parameters.AddWithValue("days", days);
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
events.Add(new SelfServeSignalEvent(
SessionId: reader.IsDBNull(0) ? null : NullIfEmpty(reader.GetString(0)),
EventType: reader.GetString(1),
Query: reader.GetString(2),
AnswerStatus: reader.IsDBNull(3) ? null : reader.GetString(3),
CreatedAt: new DateTimeOffset(reader.GetDateTime(4), TimeSpan.Zero),
OrderKey: reader.GetString(5)));
}
return events;
}
private IReadOnlyList<SelfServeSignalEvent> BuildFallbackSelfServeSignalEvents(string tenantId, TimeSpan window)
{
return _analyticsService.GetFallbackEventsSnapshot(tenantId, window)
.Where(static item =>
item.Event.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) ||
item.Event.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase) ||
item.Event.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase) ||
item.Event.EventType.Equals("click", StringComparison.OrdinalIgnoreCase))
.Select((item, index) => new SelfServeSignalEvent(
SessionId: item.Event.SessionId,
EventType: item.Event.EventType,
Query: item.Event.Query,
AnswerStatus: item.Event.AnswerStatus,
CreatedAt: item.RecordedAt,
OrderKey: index.ToString("D8", CultureInfo.InvariantCulture)))
.ToArray();
}
private static void ApplySelfServeMetrics(SearchQualityMetricsEntry metrics, IReadOnlyList<SelfServeSignalEvent> signals)
{
var totalAnswerFrames = signals.Count(static evt =>
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase));
var fallbackAnswers = signals.Count(IsFallbackAnswerEvent);
var clarifyAnswers = signals.Count(static evt =>
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) &&
string.Equals(evt.AnswerStatus, "clarify", StringComparison.OrdinalIgnoreCase));
var insufficientAnswers = signals.Count(static evt =>
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) &&
string.Equals(evt.AnswerStatus, "insufficient", StringComparison.OrdinalIgnoreCase));
metrics.FallbackAnswerRate = totalAnswerFrames == 0
? 0d
: Math.Round((double)fallbackAnswers / totalAnswerFrames * 100d, 1);
metrics.ClarifyRate = totalAnswerFrames == 0
? 0d
: Math.Round((double)clarifyAnswers / totalAnswerFrames * 100d, 1);
metrics.InsufficientRate = totalAnswerFrames == 0
? 0d
: Math.Round((double)insufficientAnswers / totalAnswerFrames * 100d, 1);
metrics.ReformulationCount = signals.Count(static evt =>
evt.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase));
metrics.RescueActionCount = signals.Count(static evt =>
evt.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase));
metrics.AbandonedFallbackCount = GetAbandonedFallbackSessions(signals).Count;
}
private static IReadOnlyList<AlertCandidate> BuildFallbackLoopCandidates(IReadOnlyList<SelfServeSignalEvent> events)
{
var rawCandidates = new List<AlertCandidate>();
foreach (var group in events
.Where(static evt => !string.IsNullOrWhiteSpace(evt.SessionId))
.GroupBy(static evt => evt.SessionId!, StringComparer.Ordinal))
{
var ordered = OrderSessionEvents(group);
var fallbackFrames = ordered.Where(IsFallbackAnswerEvent).ToArray();
if (fallbackFrames.Length < FallbackLoopAlertThreshold)
{
continue;
}
var first = fallbackFrames[0];
var last = fallbackFrames[^1];
rawCandidates.Add(new AlertCandidate(last.Query, fallbackFrames.Length, first.CreatedAt, last.CreatedAt));
}
return rawCandidates
.GroupBy(static candidate => candidate.Query, StringComparer.OrdinalIgnoreCase)
.Select(group => new AlertCandidate(
group.Key,
group.Sum(static candidate => candidate.OccurrenceCount),
group.Min(static candidate => candidate.FirstSeen),
group.Max(static candidate => candidate.LastSeen)))
.OrderByDescending(static candidate => candidate.OccurrenceCount)
.ToArray();
}
private static IReadOnlyList<AlertCandidate> BuildAbandonedFallbackCandidates(IReadOnlyList<SelfServeSignalEvent> events)
{
return GetAbandonedFallbackSessions(events)
.GroupBy(static candidate => candidate.Query, StringComparer.OrdinalIgnoreCase)
.Select(group => new AlertCandidate(
group.Key,
group.Sum(static candidate => candidate.OccurrenceCount),
group.Min(static candidate => candidate.FirstSeen),
group.Max(static candidate => candidate.LastSeen)))
.Where(static candidate => candidate.OccurrenceCount >= AbandonedFallbackAlertThreshold)
.OrderByDescending(static candidate => candidate.OccurrenceCount)
.ToArray();
}
private static IReadOnlyList<AlertCandidate> GetAbandonedFallbackSessions(IReadOnlyList<SelfServeSignalEvent> events)
{
var candidates = new List<AlertCandidate>();
foreach (var group in events
.Where(static evt => !string.IsNullOrWhiteSpace(evt.SessionId))
.GroupBy(static evt => evt.SessionId!, StringComparer.Ordinal))
{
var ordered = OrderSessionEvents(group);
var lastFallbackIndex = -1;
for (var index = ordered.Length - 1; index >= 0; index--)
{
if (IsFallbackAnswerEvent(ordered[index]))
{
lastFallbackIndex = index;
break;
}
}
if (lastFallbackIndex < 0)
{
continue;
}
var lastFallback = ordered[lastFallbackIndex];
var recovered = false;
for (var index = lastFallbackIndex + 1; index < ordered.Length; index++)
{
if (IsRecoveryEvent(ordered[index]))
{
recovered = true;
break;
}
}
if (recovered)
{
continue;
}
candidates.Add(new AlertCandidate(lastFallback.Query, 1, lastFallback.CreatedAt, lastFallback.CreatedAt));
}
return candidates;
}
private static bool IsFallbackAnswerEvent(SelfServeSignalEvent evt)
{
return evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase)
&& (string.Equals(evt.AnswerStatus, "clarify", StringComparison.OrdinalIgnoreCase)
|| string.Equals(evt.AnswerStatus, "insufficient", StringComparison.OrdinalIgnoreCase));
}
private static bool IsRecoveryEvent(SelfServeSignalEvent evt)
{
return evt.EventType.Equals("click", StringComparison.OrdinalIgnoreCase)
|| evt.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase)
|| evt.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase)
|| (evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase)
&& string.Equals(evt.AnswerStatus, "grounded", StringComparison.OrdinalIgnoreCase));
}
private static SelfServeSignalEvent[] OrderSessionEvents(IEnumerable<SelfServeSignalEvent> events)
{
return events
.OrderBy(static evt => evt.CreatedAt)
.ThenBy(static evt => evt.OrderKey, StringComparer.Ordinal)
.ToArray();
}
private static string? NullIfEmpty(string value)
{
return string.IsNullOrWhiteSpace(value) ? null : value;
}
private async Task<IReadOnlyList<AlertCandidate>> LoadZeroResultCandidatesAsync(
string tenantId,
TimeSpan window,
@@ -1170,12 +1434,26 @@ internal sealed class SearchQualityMetricsEntry
public double ZeroResultRate { get; set; }
public double AvgResultCount { get; set; }
public double FeedbackScore { get; set; }
public double FallbackAnswerRate { get; set; }
public double ClarifyRate { get; set; }
public double InsufficientRate { get; set; }
public int ReformulationCount { get; set; }
public int RescueActionCount { get; set; }
public int AbandonedFallbackCount { get; set; }
public string Period { get; set; } = "7d";
public IReadOnlyList<SearchQualityLowQualityRow> LowQualityResults { get; set; } = [];
public IReadOnlyList<SearchQualityTopQueryRow> TopQueries { get; set; } = [];
public IReadOnlyList<SearchQualityTrendPoint> Trend { get; set; } = [];
}
internal sealed record SelfServeSignalEvent(
string? SessionId,
string EventType,
string Query,
string? AnswerStatus,
DateTimeOffset CreatedAt,
string OrderKey);
internal sealed class SearchQualityLowQualityRow
{
public string EntityKey { get; set; } = string.Empty;

View File

@@ -107,18 +107,23 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
return EmptyResponse(string.Empty, request.K, "empty");
}
if (query.Length > _unifiedOptions.MaxQueryLength)
{
return EmptyResponse(query, request.K, "query_too_long", request.Ambient);
}
var tenantId = request.Filters?.Tenant ?? "global";
var userId = request.Filters?.UserId ?? "anonymous";
if (query.Length > _unifiedOptions.MaxQueryLength)
{
var earlyResponse = EmptyResponse(query, request.K, "query_too_long", request.Ambient);
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
return earlyResponse;
}
var tenantFlags = ResolveTenantFeatureFlags(tenantId);
if (!_options.Enabled || !IsSearchEnabledForTenant(tenantFlags) || string.IsNullOrWhiteSpace(_options.ConnectionString))
{
return EmptyResponse(query, request.K, "disabled", request.Ambient);
var earlyResponse = EmptyResponse(query, request.K, "disabled", request.Ambient);
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
return earlyResponse;
}
if (request.Ambient?.ResetSession == true &&
@@ -321,6 +326,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
refinements,
contextAnswer);
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan, response, cancellationToken).ConfigureAwait(false);
EmitTelemetry(plan, response, tenantId);
return response;
}
@@ -1858,4 +1864,40 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
HasSuggestions: response.Suggestions is { Count: > 0 },
HasRefinements: response.Refinements is { Count: > 0 }));
}
private async Task RecordAnswerFrameAnalyticsAsync(
string tenantId,
string userId,
UnifiedSearchRequest request,
QueryPlan? plan,
UnifiedSearchResponse response,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(response.Query))
{
return;
}
var contextAnswer = response.ContextAnswer;
if (contextAnswer is null)
{
return;
}
var domain = ResolveContextDomain(plan, response.Cards, request.Ambient)
?? response.Cards.FirstOrDefault()?.Domain
?? request.Filters?.Domains?.FirstOrDefault();
await _analyticsService.RecordEventAsync(new SearchAnalyticsEvent(
TenantId: tenantId,
EventType: "answer_frame",
Query: response.Query,
UserId: userId,
SessionId: request.Ambient?.SessionId,
Domain: domain,
ResultCount: response.Cards.Count,
DurationMs: (int)Math.Min(int.MaxValue, Math.Max(0, response.Diagnostics.DurationMs)),
AnswerStatus: contextAnswer.Status,
AnswerCode: contextAnswer.Code), cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -1511,6 +1511,12 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
"zero-result rate must be computed from raw zero_result analytics events");
metrics.AvgResultCount.Should().BeApproximately(2.0, 0.2);
metrics.FeedbackScore.Should().BeApproximately(50.0, 0.2);
metrics.FallbackAnswerRate.Should().Be(0);
metrics.ClarifyRate.Should().Be(0);
metrics.InsufficientRate.Should().Be(0);
metrics.ReformulationCount.Should().Be(0);
metrics.RescueActionCount.Should().Be(0);
metrics.AbandonedFallbackCount.Should().Be(0);
}
[Fact]
@@ -1579,6 +1585,74 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
.Should().BeTrue("low-quality rows should include repeated negative feedback entities");
}
[Fact]
public async Task G10_SelfServeMetrics_IncludeFallbackReformulationAndRescueSignals()
{
var tenant = $"selfserve-metrics-{Guid.NewGuid():N}";
using var writer = CreateAuthenticatedClient(tenant);
using var admin = CreateAdminClient(tenant);
var analyticsResponse = await writer.PostAsJsonAsync("/v1/advisory-ai/search/analytics", new SearchAnalyticsApiRequest
{
Events =
[
new SearchAnalyticsApiEvent
{
EventType = "query",
Query = "database connectivity",
ResultCount = 0
},
new SearchAnalyticsApiEvent
{
EventType = "answer_frame",
Query = "database connectivity",
SessionId = "session-selfserve-1",
Domain = "knowledge",
AnswerStatus = "clarify",
AnswerCode = "query_needs_scope"
},
new SearchAnalyticsApiEvent
{
EventType = "reformulation",
Query = "postgresql connectivity",
SessionId = "session-selfserve-1",
Domain = "knowledge"
},
new SearchAnalyticsApiEvent
{
EventType = "answer_frame",
Query = "postgresql connectivity",
SessionId = "session-selfserve-1",
Domain = "knowledge",
AnswerStatus = "insufficient",
AnswerCode = "no_grounded_evidence"
},
new SearchAnalyticsApiEvent
{
EventType = "rescue_action",
Query = "postgresql connectivity",
SessionId = "session-selfserve-1",
Domain = "knowledge"
}
]
});
analyticsResponse.StatusCode.Should().Be(HttpStatusCode.NoContent);
var metricsResponse = await admin.GetAsync("/v1/advisory-ai/search/quality/metrics?period=7d");
metricsResponse.StatusCode.Should().Be(HttpStatusCode.OK);
var metrics = await metricsResponse.Content.ReadFromJsonAsync<SearchQualityMetricsDto>();
metrics.Should().NotBeNull();
metrics!.FallbackAnswerRate.Should().BeApproximately(100.0, 0.2);
metrics.ClarifyRate.Should().BeApproximately(50.0, 0.2);
metrics.InsufficientRate.Should().BeApproximately(50.0, 0.2);
metrics.ReformulationCount.Should().Be(1);
metrics.RescueActionCount.Should().Be(1);
metrics.AbandonedFallbackCount.Should().Be(0,
"a rescue action in the same session means the fallback flow was not abandoned");
}
[Fact]
public async Task G10_AnalyticsEndpoint_ValidBatch_ReturnsNoContent()
{
@@ -1624,6 +1698,7 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
EventType: "query",
Query: query,
UserId: userId,
SessionId: "session-privacy-1",
ResultCount: 1));
var events = analyticsService.GetFallbackEventsSnapshot(tenant, TimeSpan.FromMinutes(1));
@@ -1634,6 +1709,7 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
stored.Query.Should().NotContain("alice", "raw query text must not be stored in analytics events");
stored.UserId.Should().Be(SearchAnalyticsPrivacy.HashUserId(tenant, userId));
stored.UserId.Should().NotBe(userId, "user identifiers must be pseudonymized before persistence");
stored.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId(tenant, "session-privacy-1"));
}
[Fact]
@@ -1862,6 +1938,76 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
.Should().BeTrue("five repeated zero-result events should create a zero_result quality alert");
}
[Fact]
public async Task G10_FallbackLoopSessions_CreateQualityAlert()
{
var tenant = $"fallback-loop-{Guid.NewGuid():N}";
using var scope = _factory.Services.CreateScope();
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
await analytics.RecordEventsAsync(
[
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate", SessionId: "sess-loop-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate release", SessionId: "sess-loop-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate release production", SessionId: "sess-loop-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence")
]);
await monitor.RefreshAlertsAsync(tenant);
var alerts = await monitor.GetAlertsAsync(tenant, status: "open", alertType: "fallback_loop");
var expectedQueryHash = SearchAnalyticsPrivacy.HashQuery("policy gate release production");
alerts.Any(alert => alert.Query.Equals(expectedQueryHash, StringComparison.OrdinalIgnoreCase))
.Should().BeTrue("three fallback answer frames in the same session should create a fallback_loop alert");
}
[Fact]
public async Task G10_AbandonedFallbackSessions_CreateQualityAlert()
{
var tenant = $"abandoned-fallback-{Guid.NewGuid():N}";
using var scope = _factory.Services.CreateScope();
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
await analytics.RecordEventsAsync(
[
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-2", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-3", AnswerStatus: "clarify", AnswerCode: "query_needs_scope")
]);
await monitor.RefreshAlertsAsync(tenant);
var alerts = await monitor.GetAlertsAsync(tenant, status: "open", alertType: "abandoned_fallback");
var expectedQueryHash = SearchAnalyticsPrivacy.HashQuery("unknown control token");
alerts.Any(alert => alert.Query.Equals(expectedQueryHash, StringComparison.OrdinalIgnoreCase))
.Should().BeTrue("repeated sessions that end in fallback without recovery should create an abandoned_fallback alert");
}
[Fact]
public async Task G10_RecoveredFallbackSessions_DoNotCountAsAbandoned()
{
var tenant = $"recovered-fallback-{Guid.NewGuid():N}";
using var scope = _factory.Services.CreateScope();
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
await analytics.RecordEventsAsync(
[
new SearchAnalyticsEvent(tenant, "answer_frame", "database connectivity", SessionId: "sess-recovered-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
new SearchAnalyticsEvent(tenant, "reformulation", "postgresql connectivity", SessionId: "sess-recovered-1"),
new SearchAnalyticsEvent(tenant, "answer_frame", "postgresql connectivity", SessionId: "sess-recovered-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
new SearchAnalyticsEvent(tenant, "rescue_action", "postgresql connectivity", SessionId: "sess-recovered-1")
]);
var metrics = await monitor.GetMetricsAsync(tenant, "7d");
metrics.ReformulationCount.Should().Be(1);
metrics.RescueActionCount.Should().Be(1);
metrics.AbandonedFallbackCount.Should().Be(0,
"a session with a reformulation and rescue action after fallback is a recovery path, not an abandonment");
}
[Fact]
public async Task G10_NegativeFeedbackBurst_CreatesHighNegativeFeedbackAlert()
{

View File

@@ -22,4 +22,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| SPRINT_20260224_G1-004-BENCH | DONE | Semantic recall benchmark: 13 tests in `SemanticRecallBenchmarkTests.cs`, 48-query fixture (`semantic-recall-benchmark.json`), `SemanticRecallBenchmarkStore` (33 chunks), `SemanticSimulationEncoder` (40+ semantic groups). Semantic strictly outperforms hash on synonym queries. |
| AI-SELF-005 | DONE | Integration coverage now asserts grounded, clarify, and insufficient contextual-answer states through the real endpoint contract. |
| AI-SELF-004 | DONE | Targeted telemetry coverage now verifies fallback metrics, recovered-session suppression of abandoned counts, session hashing, and self-serve alert generation with xUnit v3-compatible runner commands. |
| AI-SELF-006 | DONE | Verification includes a real local corpus rebuild and a live query assertion, not only test doubles. |

View File

@@ -8,6 +8,7 @@ using StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
using StellaOps.AdvisoryAI.UnifiedSearch.QueryUnderstanding;
using StellaOps.AdvisoryAI.UnifiedSearch.Synthesis;
using StellaOps.AdvisoryAI.Vectorization;
using System.Linq;
using System.Text.Json;
using Xunit;
@@ -124,6 +125,101 @@ public sealed class UnifiedSearchServiceTests
result.ContextAnswer.Questions!.Should().OnlyContain(question => question.Kind == "recover");
}
[Fact]
public async Task SearchAsync_records_answer_frame_analytics_for_clarify_state()
{
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
{
Enabled = true,
ConnectionString = "Host=localhost;Database=test"
});
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
var service = CreateService(
storeMock: CreateEmptyStoreMock(),
analyticsService: analyticsService,
qualityMonitor: qualityMonitor);
var result = await service.SearchAsync(
new UnifiedSearchRequest(
"status",
Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" },
Ambient: new AmbientContext
{
CurrentRoute = "/ops/operations/doctor",
SessionId = "session-123"
}),
CancellationToken.None);
result.ContextAnswer.Should().NotBeNull();
result.ContextAnswer!.Status.Should().Be("clarify");
var events = analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1));
events.Should().ContainSingle();
var answerFrame = events.Single().Event;
answerFrame.EventType.Should().Be("answer_frame");
answerFrame.AnswerStatus.Should().Be("clarify");
answerFrame.AnswerCode.Should().Be(result.ContextAnswer.Code);
answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-123"));
answerFrame.Query.Should().Be(SearchAnalyticsPrivacy.HashQuery("status"));
}
[Fact]
public async Task SearchAsync_records_answer_frame_analytics_for_grounded_state()
{
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
{
Enabled = true,
ConnectionString = "Host=localhost;Database=test"
});
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
var doctorRow = MakeRow(
"chunk-doctor-analytics",
"doctor_check",
"PostgreSQL connectivity",
JsonDocument.Parse("{\"domain\":\"knowledge\",\"checkCode\":\"check.core.db.connectivity\"}"));
var storeMock = new Mock<IKnowledgeSearchStore>();
storeMock.Setup(s => s.SearchFtsAsync(
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>(), It.IsAny<string?>()))
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow });
storeMock.Setup(s => s.LoadVectorCandidatesAsync(
It.IsAny<float[]>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
.ReturnsAsync([]);
var service = CreateService(
storeMock: storeMock,
analyticsService: analyticsService,
qualityMonitor: qualityMonitor);
var result = await service.SearchAsync(
new UnifiedSearchRequest(
"database connectivity",
Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" },
Ambient: new AmbientContext
{
CurrentRoute = "/ops/operations/doctor",
SessionId = "session-456"
}),
CancellationToken.None);
result.ContextAnswer.Should().NotBeNull();
result.ContextAnswer!.Status.Should().Be("grounded");
var events = analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1));
events.Should().ContainSingle();
var answerFrame = events.Single().Event;
answerFrame.EventType.Should().Be("answer_frame");
answerFrame.AnswerStatus.Should().Be("grounded");
answerFrame.AnswerCode.Should().Be("retrieved_evidence");
answerFrame.ResultCount.Should().Be(1);
answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-456"));
}
[Fact]
public async Task SearchAsync_returns_empty_when_tenant_feature_flag_disables_search()
{
@@ -827,7 +923,9 @@ public sealed class UnifiedSearchServiceTests
private static UnifiedSearchService CreateService(
bool enabled = true,
Mock<IKnowledgeSearchStore>? storeMock = null,
UnifiedSearchOptions? unifiedOptions = null)
UnifiedSearchOptions? unifiedOptions = null,
SearchAnalyticsService? analyticsService = null,
SearchQualityMonitor? qualityMonitor = null)
{
var options = Options.Create(new KnowledgeSearchOptions
{
@@ -855,8 +953,8 @@ public sealed class UnifiedSearchServiceTests
var weightCalculator = new DomainWeightCalculator(extractor, classifier, options);
var planBuilder = new QueryPlanBuilder(extractor, classifier, weightCalculator);
var synthesisEngine = new SynthesisTemplateEngine();
var analyticsService = new SearchAnalyticsService(options, NullLogger<SearchAnalyticsService>.Instance);
var qualityMonitor = new SearchQualityMonitor(options, NullLogger<SearchQualityMonitor>.Instance);
analyticsService ??= new SearchAnalyticsService(options, NullLogger<SearchAnalyticsService>.Instance);
qualityMonitor ??= new SearchQualityMonitor(options, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
var entityAliasService = new Mock<IEntityAliasService>();
entityAliasService.Setup(s => s.ResolveAliasesAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(Array.Empty<(string EntityKey, string EntityType)>());