diff --git a/docs/implplan/SPRINT_20260307_005_AdvisoryAI_grounded_search_answer_orchestration.md b/docs/implplan/SPRINT_20260307_005_AdvisoryAI_grounded_search_answer_orchestration.md index b2daf8df5..f2f91abe1 100644 --- a/docs/implplan/SPRINT_20260307_005_AdvisoryAI_grounded_search_answer_orchestration.md +++ b/docs/implplan/SPRINT_20260307_005_AdvisoryAI_grounded_search_answer_orchestration.md @@ -65,7 +65,7 @@ Completion criteria: - [ ] Ambiguous queries return clarifying prompts instead of a blank answer slot. ### AI-SELF-004 - Self-serve telemetry and gap surfacing -Status: TODO +Status: DONE Dependency: AI-SELF-002 Owners: Developer (AdvisoryAI), Test Automation Task description: @@ -73,9 +73,9 @@ Task description: - Expose enough structured data to drive a gap-closure backlog. Completion criteria: -- [ ] Telemetry captures unanswered and reformulated journeys without persisting raw sensitive prompts unnecessarily. -- [ ] Operational docs explain how to review self-serve gaps. -- [ ] Tests cover telemetry emission for fallback paths. +- [x] Telemetry captures unanswered and reformulated journeys without persisting raw sensitive prompts unnecessarily. +- [x] Operational docs explain how to review self-serve gaps. +- [x] Tests cover telemetry emission for fallback paths. ### AI-SELF-005 - Targeted behavioral verification Status: DONE @@ -110,6 +110,8 @@ Completion criteria: | 2026-03-07 | Implemented `contextAnswer` in unified search/backend API mapping, added deterministic `grounded` / `clarify` / `insufficient` rules plus follow-up question generation, and extended telemetry fields for answer-state visibility. | Developer | | 2026-03-07 | Verified the AdvisoryAI test project after the contract change with `dotnet test "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj" --no-restore -v normal` (`877/877` passing). | Test Automation | | 2026-03-07 | Exercised the live rebuilt-corpus lane against `http://127.0.0.1:10451`: `POST /v1/advisory-ai/index/rebuild`, `POST /v1/search/index/rebuild`, then `POST /v1/search/query` for `database connectivity`, which returned `contextAnswer.status = grounded`, 3 citations, and 10 cards over ingested data. | Test Automation | +| 2026-03-07 | Implemented optional self-serve analytics columns (`session_id`, `answer_status`, `answer_code`), metrics (`fallbackAnswerRate`, `clarifyRate`, `insufficientRate`, `reformulationCount`, `rescueActionCount`, `abandonedFallbackCount`), and alerting (`fallback_loop`, `abandoned_fallback`) with privacy-preserving session hashing. | Developer | +| 2026-03-07 | Verified targeted telemetry coverage with xUnit v3 runner commands against `UnifiedSearchSprintIntegrationTests` and added a recovered-session regression test so rescue actions clear abandonment as expected. | Test Automation | ## Decisions & Risks - Decision: the backend contract must return explicit answer states instead of leaving the UI to infer confidence from cards alone. @@ -122,6 +124,8 @@ Completion criteria: - Risk: mocked endpoint tests can overstate confidence if ingestion adapters or corpus rebuild order drift. - Mitigation: keep rebuild order documented, execute it during verification, and record which routes have live-ingested parity. - Decision: `stella advisoryai sources prepare` is optional for local verification when checked-in Doctor seed/control files are already sufficient, but it requires `STELLAOPS_BACKEND_URL` whenever live Doctor discovery is expected. +- Decision: self-serve analytics is optional and additive; search behavior must not depend on telemetry event emission. +- Decision: targeted AdvisoryAI verification uses xUnit v3 / Microsoft.Testing.Platform-compatible filters (`-- --filter-class` or the built test executable), not legacy VSTest `--filter`. ## Next Checkpoints - 2026-03-10: Freeze answer payload shape and fallback taxonomy. diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchAnalyticsEndpoints.cs b/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchAnalyticsEndpoints.cs index bc85d1639..a1fdce32c 100644 --- a/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchAnalyticsEndpoints.cs +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchAnalyticsEndpoints.cs @@ -17,7 +17,10 @@ public static class SearchAnalyticsEndpoints "query", "click", "zero_result", - "synthesis" + "synthesis", + "answer_frame", + "reformulation", + "rescue_action" }; public static RouteGroupBuilder MapSearchAnalyticsEndpoints(this IEndpointRouteBuilder builder) @@ -30,10 +33,11 @@ public static class SearchAnalyticsEndpoints group.MapPost("/analytics", RecordAnalyticsAsync) .WithName("SearchAnalyticsRecord") - .WithSummary("Records batch search analytics events (query, click, zero_result, synthesis).") + .WithSummary("Records batch search analytics events (query, click, zero_result, synthesis, answer_frame, reformulation, rescue_action).") .WithDescription( "Accepts a batch of search analytics events for tracking query frequency, click-through rates, " + - "zero-result queries, and synthesis usage. Queries and user identifiers are pseudonymized before persistence. " + + "zero-result queries, self-serve answer states, reformulations, rescue-action usage, and synthesis usage. " + + "Queries and user/session identifiers are pseudonymized before persistence. " + "Fire-and-forget from the client; failures do not affect search functionality.") .RequireAuthorization(AdvisoryAIPolicies.OperatePolicy) .Produces(StatusCodes.Status204NoContent) @@ -113,11 +117,14 @@ public static class SearchAnalyticsEndpoints EventType: apiEvent.EventType.Trim().ToLowerInvariant(), Query: apiEvent.Query.Trim(), UserId: userId, + SessionId: string.IsNullOrWhiteSpace(apiEvent.SessionId) ? null : apiEvent.SessionId.Trim(), EntityKey: string.IsNullOrWhiteSpace(apiEvent.EntityKey) ? null : apiEvent.EntityKey.Trim(), Domain: string.IsNullOrWhiteSpace(apiEvent.Domain) ? null : apiEvent.Domain.Trim(), ResultCount: apiEvent.ResultCount, Position: apiEvent.Position, - DurationMs: apiEvent.DurationMs)); + DurationMs: apiEvent.DurationMs, + AnswerStatus: string.IsNullOrWhiteSpace(apiEvent.AnswerStatus) ? null : apiEvent.AnswerStatus.Trim(), + AnswerCode: string.IsNullOrWhiteSpace(apiEvent.AnswerCode) ? null : apiEvent.AnswerCode.Trim())); } if (events.Count > 0) @@ -301,6 +308,8 @@ public sealed record SearchAnalyticsApiEvent public string Query { get; init; } = string.Empty; + public string? SessionId { get; init; } + public string? EntityKey { get; init; } public string? Domain { get; init; } @@ -310,6 +319,10 @@ public sealed record SearchAnalyticsApiEvent public int? Position { get; init; } public int? DurationMs { get; init; } + + public string? AnswerStatus { get; init; } + + public string? AnswerCode { get; init; } } public sealed record SearchHistoryApiResponse diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchFeedbackEndpoints.cs b/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchFeedbackEndpoints.cs index e480bbd43..68b65c3ed 100644 --- a/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchFeedbackEndpoints.cs +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/Endpoints/SearchFeedbackEndpoints.cs @@ -39,11 +39,11 @@ public static class SearchFeedbackEndpoints // G10-002: List quality alerts (admin only) group.MapGet("/quality/alerts", GetAlertsAsync) .WithName("SearchQualityAlertsList") - .WithSummary("Lists open search quality alerts (zero-result queries, high negative feedback).") + .WithSummary("Lists open search quality alerts (zero-result, high negative feedback, fallback loops, abandoned fallback).") .WithDescription( "Returns search quality alerts ordered by occurrence count. " + "Filterable by status (open, acknowledged, resolved) and alert type " + - "(zero_result, low_feedback, high_negative_feedback). Requires admin scope.") + "(zero_result, high_negative_feedback, fallback_loop, abandoned_fallback). Requires admin scope.") .RequireAuthorization(AdvisoryAIPolicies.AdminPolicy) .Produces>(StatusCodes.Status200OK) .Produces(StatusCodes.Status403Forbidden); @@ -198,6 +198,12 @@ public static class SearchFeedbackEndpoints ZeroResultRate = metrics.ZeroResultRate, AvgResultCount = metrics.AvgResultCount, FeedbackScore = metrics.FeedbackScore, + FallbackAnswerRate = metrics.FallbackAnswerRate, + ClarifyRate = metrics.ClarifyRate, + InsufficientRate = metrics.InsufficientRate, + ReformulationCount = metrics.ReformulationCount, + RescueActionCount = metrics.RescueActionCount, + AbandonedFallbackCount = metrics.AbandonedFallbackCount, Period = metrics.Period, LowQualityResults = metrics.LowQualityResults .Select(row => new SearchLowQualityResultDto @@ -308,6 +314,12 @@ public sealed record SearchQualityMetricsDto public double ZeroResultRate { get; init; } public double AvgResultCount { get; init; } public double FeedbackScore { get; init; } + public double FallbackAnswerRate { get; init; } + public double ClarifyRate { get; init; } + public double InsufficientRate { get; init; } + public int ReformulationCount { get; init; } + public int RescueActionCount { get; init; } + public int AbandonedFallbackCount { get; init; } public string Period { get; init; } = "7d"; public IReadOnlyList LowQualityResults { get; init; } = []; public IReadOnlyList TopQueries { get; init; } = []; diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/TASKS.md b/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/TASKS.md index afd54eaaa..04429beee 100644 --- a/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/TASKS.md +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI.WebService/TASKS.md @@ -17,4 +17,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol | SPRINT_20260224_G1-G10 | DONE | Search improvement sprints G1–G10 implemented. New endpoints: `SearchAnalyticsEndpoints.cs` (history, events, popularity), `SearchFeedbackEndpoints.cs` (feedback, quality alerts, metrics). Extended: `UnifiedSearchEndpoints.cs` (suggestions, refinements, previews, diagnostics.activeEncoder). Extended: `KnowledgeSearchEndpoints.cs` (activeEncoder in diagnostics). See `docs/modules/advisory-ai/knowledge-search.md` for full testing guide. | | AI-SELF-001 | DONE | Unified search endpoint contract now exposes backend contextual answer fields for self-serve search. | +| AI-SELF-004 | DONE | Search analytics and quality endpoints now surface self-serve metrics and alerts (`fallback_loop`, `abandoned_fallback`) while keeping telemetry optional. | | AI-SELF-006 | DONE | Endpoint readiness now includes a proven local rebuilt-corpus verification lane in addition to stubbed integration tests. | diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI/Storage/Migrations/008_search_self_serve_analytics.sql b/src/AdvisoryAI/StellaOps.AdvisoryAI/Storage/Migrations/008_search_self_serve_analytics.sql new file mode 100644 index 000000000..a4fcd72ad --- /dev/null +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI/Storage/Migrations/008_search_self_serve_analytics.sql @@ -0,0 +1,20 @@ +-- 008_search_self_serve_analytics.sql +-- Adds self-serve telemetry fields to search_events so fallback loops, +-- reformulations, rescue actions, and abandoned fallback sessions can be tracked. + +ALTER TABLE advisoryai.search_events + ADD COLUMN IF NOT EXISTS session_id TEXT; + +ALTER TABLE advisoryai.search_events + ADD COLUMN IF NOT EXISTS answer_status TEXT; + +ALTER TABLE advisoryai.search_events + ADD COLUMN IF NOT EXISTS answer_code TEXT; + +CREATE INDEX IF NOT EXISTS idx_search_events_tenant_session + ON advisoryai.search_events (tenant_id, session_id, created_at) + WHERE session_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_search_events_answer_status + ON advisoryai.search_events (tenant_id, answer_status, created_at) + WHERE answer_status IS NOT NULL; diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI/TASKS.md b/src/AdvisoryAI/StellaOps.AdvisoryAI/TASKS.md index 1af18ce70..cef98a132 100644 --- a/src/AdvisoryAI/StellaOps.AdvisoryAI/TASKS.md +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI/TASKS.md @@ -13,7 +13,7 @@ Source of truth: `docs/implplan/SPRINT_20260113_005_ADVISORYAI_controlled_conver | AI-SELF-001 | DONE | Unified search now emits the contextual answer payload (`contextAnswer`) with answer state, citations, and follow-up questions for self-serve search. | | AI-SELF-002 | DONE | Deterministic grounded/clarify/insufficient fallback policy is implemented in unified search orchestration. | | AI-SELF-003 | DONE | Follow-up question generation from route/domain intent, recent actions, and evidence is implemented. | -| AI-SELF-004 | TODO | Telemetry for unanswered and reformulated self-serve journeys is still pending. | +| AI-SELF-004 | DONE | Optional self-serve telemetry now captures answer frames, reformulations, rescue actions, and abandonment signals via hashed session ids; quality metrics and alerts expose the gaps operationally. | | AI-SELF-006 | DONE | Live ingestion-backed answer verification succeeded on the Doctor/knowledge route after local rebuild. | | SPRINT_20260222_051-AKS-INGEST | DONE | Added deterministic AKS ingestion controls: markdown allow-list manifest loading, OpenAPI aggregate source path support, and doctor control projection integration for search chunks, including fallback doctor metadata hydration from controls projection fields. | | AUDIT-0017-M | DONE | Maintainability audit for StellaOps.AdvisoryAI. | diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsPrivacy.cs b/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsPrivacy.cs index 4c635b9d2..fc9cf6461 100644 --- a/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsPrivacy.cs +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsPrivacy.cs @@ -32,6 +32,19 @@ internal static class SearchAnalyticsPrivacy return Convert.ToHexString(hash).ToLowerInvariant(); } + public static string? HashSessionId(string tenantId, string? sessionId) + { + if (string.IsNullOrWhiteSpace(sessionId)) + { + return null; + } + + var normalizedTenant = tenantId.Trim().ToLowerInvariant(); + var normalizedSession = sessionId.Trim().ToLowerInvariant(); + var hash = SHA256.HashData(Encoding.UTF8.GetBytes($"{normalizedTenant}|session|{normalizedSession}")); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + public static string? RedactFreeform(string? value) { _ = value; diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsService.cs b/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsService.cs index c0357eae1..f73e5a0fe 100644 --- a/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsService.cs +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchAnalyticsService.cs @@ -37,11 +37,14 @@ internal sealed class SearchAnalyticsService await conn.OpenAsync(ct).ConfigureAwait(false); await using var cmd = new NpgsqlCommand(@" - INSERT INTO advisoryai.search_events (tenant_id, user_id, event_type, query, entity_key, domain, result_count, position, duration_ms) - VALUES (@tenant_id, @user_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms)", conn); + INSERT INTO advisoryai.search_events + (tenant_id, user_id, session_id, event_type, query, entity_key, domain, result_count, position, duration_ms, answer_status, answer_code) + VALUES + (@tenant_id, @user_id, @session_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms, @answer_status, @answer_code)", conn); cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId); cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value); + cmd.Parameters.AddWithValue("session_id", (object?)persistedEvent.SessionId ?? DBNull.Value); cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType); cmd.Parameters.AddWithValue("query", persistedEvent.Query); cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value); @@ -49,6 +52,8 @@ internal sealed class SearchAnalyticsService cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value); cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value); cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value); + cmd.Parameters.AddWithValue("answer_status", (object?)persistedEvent.AnswerStatus ?? DBNull.Value); + cmd.Parameters.AddWithValue("answer_code", (object?)persistedEvent.AnswerCode ?? DBNull.Value); await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); RecordFallbackEvent(persistedEvent, recordedAt); @@ -88,11 +93,14 @@ internal sealed class SearchAnalyticsService var persistedEvent = SanitizeEvent(evt); await using var cmd = new NpgsqlCommand(@" - INSERT INTO advisoryai.search_events (tenant_id, user_id, event_type, query, entity_key, domain, result_count, position, duration_ms) - VALUES (@tenant_id, @user_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms)", conn); + INSERT INTO advisoryai.search_events + (tenant_id, user_id, session_id, event_type, query, entity_key, domain, result_count, position, duration_ms, answer_status, answer_code) + VALUES + (@tenant_id, @user_id, @session_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms, @answer_status, @answer_code)", conn); cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId); cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value); + cmd.Parameters.AddWithValue("session_id", (object?)persistedEvent.SessionId ?? DBNull.Value); cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType); cmd.Parameters.AddWithValue("query", persistedEvent.Query); cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value); @@ -100,6 +108,8 @@ internal sealed class SearchAnalyticsService cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value); cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value); cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value); + cmd.Parameters.AddWithValue("answer_status", (object?)persistedEvent.AnswerStatus ?? DBNull.Value); + cmd.Parameters.AddWithValue("answer_code", (object?)persistedEvent.AnswerCode ?? DBNull.Value); await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); RecordFallbackEvent(persistedEvent, recordedAt); @@ -637,8 +647,16 @@ internal sealed class SearchAnalyticsService { return evt with { + EventType = evt.EventType.Trim().ToLowerInvariant(), Query = SearchAnalyticsPrivacy.HashQuery(evt.Query), - UserId = SearchAnalyticsPrivacy.HashUserId(evt.TenantId, evt.UserId) + UserId = SearchAnalyticsPrivacy.HashUserId(evt.TenantId, evt.UserId), + SessionId = SearchAnalyticsPrivacy.HashSessionId(evt.TenantId, evt.SessionId), + AnswerStatus = string.IsNullOrWhiteSpace(evt.AnswerStatus) + ? null + : evt.AnswerStatus.Trim().ToLowerInvariant(), + AnswerCode = string.IsNullOrWhiteSpace(evt.AnswerCode) + ? null + : evt.AnswerCode.Trim().ToLowerInvariant() }; } @@ -667,11 +685,14 @@ internal record SearchAnalyticsEvent( string EventType, string Query, string? UserId = null, + string? SessionId = null, string? EntityKey = null, string? Domain = null, int? ResultCount = null, int? Position = null, - int? DurationMs = null); + int? DurationMs = null, + string? AnswerStatus = null, + string? AnswerCode = null); internal record SearchHistoryEntry( string HistoryId, diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchQualityMonitor.cs b/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchQualityMonitor.cs index d17cd7bcd..3d1b732c5 100644 --- a/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchQualityMonitor.cs +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/Analytics/SearchQualityMonitor.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Npgsql; @@ -6,7 +7,8 @@ using StellaOps.AdvisoryAI.KnowledgeSearch; namespace StellaOps.AdvisoryAI.UnifiedSearch.Analytics; /// -/// Monitors search quality by analysing feedback data and zero-result queries. +/// Monitors search quality by analysing feedback data, zero-result queries, +/// and self-serve fallback telemetry. /// Provides CRUD for search_quality_alerts and search_feedback tables. /// Sprint: SPRINT_20260224_110 (G10-001, G10-002) /// @@ -17,6 +19,8 @@ internal sealed class SearchQualityMonitor private const int DefaultAlertWindowDays = 7; private const int ZeroResultAlertThreshold = 3; private const int NegativeFeedbackAlertThreshold = 3; + private const int FallbackLoopAlertThreshold = 3; + private const int AbandonedFallbackAlertThreshold = 3; private readonly KnowledgeSearchOptions _options; private readonly ILogger _logger; @@ -175,6 +179,32 @@ internal sealed class SearchQualityMonitor candidate.LastSeen, ct).ConfigureAwait(false); } + + var selfServeSignals = await LoadSelfServeSignalEventsAsync(tenantId, window, ct).ConfigureAwait(false); + + foreach (var candidate in BuildFallbackLoopCandidates(selfServeSignals)) + { + await UpsertAlertAsync( + tenantId, + alertType: "fallback_loop", + candidate.Query, + candidate.OccurrenceCount, + candidate.FirstSeen, + candidate.LastSeen, + ct).ConfigureAwait(false); + } + + foreach (var candidate in BuildAbandonedFallbackCandidates(selfServeSignals)) + { + await UpsertAlertAsync( + tenantId, + alertType: "abandoned_fallback", + candidate.Query, + candidate.OccurrenceCount, + candidate.FirstSeen, + candidate.LastSeen, + ct).ConfigureAwait(false); + } } public async Task> GetAlertsAsync( @@ -423,6 +453,9 @@ internal sealed class SearchQualityMonitor } await feedbackReader.CloseAsync().ConfigureAwait(false); + var selfServeSignals = await LoadSelfServeSignalEventsAsync(conn, tenantId, days, ct).ConfigureAwait(false); + ApplySelfServeMetrics(metrics, selfServeSignals); + metrics.LowQualityResults = await LoadLowQualityResultsAsync(conn, tenantId, days, ct).ConfigureAwait(false); metrics.TopQueries = await LoadTopQueriesAsync(conn, tenantId, days, ct).ConfigureAwait(false); metrics.Trend = await LoadTrendPointsAsync(conn, tenantId, ct).ConfigureAwait(false); @@ -473,7 +506,8 @@ internal sealed class SearchQualityMonitor ? 0d : (double)helpfulCount / feedbackSignals.Length * 100d; - return new SearchQualityMetricsEntry + var selfServeSignals = BuildFallbackSelfServeSignalEvents(tenantId, window); + var metrics = new SearchQualityMetricsEntry { TotalSearches = totalSearches, ZeroResultRate = totalSearches == 0 ? 0d : Math.Round((double)zeroResults / totalSearches * 100d, 1), @@ -484,6 +518,10 @@ internal sealed class SearchQualityMonitor TopQueries = BuildFallbackTopQueries(tenantId, window), Trend = BuildFallbackTrendPoints(tenantId), }; + + ApplySelfServeMetrics(metrics, selfServeSignals); + + return metrics; } private async Task> LoadLowQualityResultsAsync( @@ -758,6 +796,232 @@ internal sealed class SearchQualityMonitor return points; } + private async Task> LoadSelfServeSignalEventsAsync( + string tenantId, + TimeSpan window, + CancellationToken ct) + { + var days = Math.Max(1, (int)Math.Ceiling(window.TotalDays)); + + if (!string.IsNullOrWhiteSpace(_options.ConnectionString)) + { + try + { + await using var conn = new NpgsqlConnection(_options.ConnectionString); + await conn.OpenAsync(ct).ConfigureAwait(false); + return await LoadSelfServeSignalEventsAsync(conn, tenantId, days, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Failed to load self-serve signal events from database."); + } + } + + return BuildFallbackSelfServeSignalEvents(tenantId, window); + } + + private static async Task> LoadSelfServeSignalEventsAsync( + NpgsqlConnection conn, + string tenantId, + int days, + CancellationToken ct) + { + var events = new List(); + await using var cmd = new NpgsqlCommand(@" + SELECT + COALESCE(session_id, ''), + event_type, + query, + answer_status, + created_at, + event_id::text + FROM advisoryai.search_events + WHERE tenant_id = @tenant_id + AND created_at > now() - make_interval(days => @days) + AND event_type IN ('answer_frame', 'reformulation', 'rescue_action', 'click') + ORDER BY created_at ASC, event_id ASC", conn); + cmd.Parameters.AddWithValue("tenant_id", tenantId); + cmd.Parameters.AddWithValue("days", days); + + await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + events.Add(new SelfServeSignalEvent( + SessionId: reader.IsDBNull(0) ? null : NullIfEmpty(reader.GetString(0)), + EventType: reader.GetString(1), + Query: reader.GetString(2), + AnswerStatus: reader.IsDBNull(3) ? null : reader.GetString(3), + CreatedAt: new DateTimeOffset(reader.GetDateTime(4), TimeSpan.Zero), + OrderKey: reader.GetString(5))); + } + + return events; + } + + private IReadOnlyList BuildFallbackSelfServeSignalEvents(string tenantId, TimeSpan window) + { + return _analyticsService.GetFallbackEventsSnapshot(tenantId, window) + .Where(static item => + item.Event.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) || + item.Event.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase) || + item.Event.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase) || + item.Event.EventType.Equals("click", StringComparison.OrdinalIgnoreCase)) + .Select((item, index) => new SelfServeSignalEvent( + SessionId: item.Event.SessionId, + EventType: item.Event.EventType, + Query: item.Event.Query, + AnswerStatus: item.Event.AnswerStatus, + CreatedAt: item.RecordedAt, + OrderKey: index.ToString("D8", CultureInfo.InvariantCulture))) + .ToArray(); + } + + private static void ApplySelfServeMetrics(SearchQualityMetricsEntry metrics, IReadOnlyList signals) + { + var totalAnswerFrames = signals.Count(static evt => + evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase)); + var fallbackAnswers = signals.Count(IsFallbackAnswerEvent); + var clarifyAnswers = signals.Count(static evt => + evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) && + string.Equals(evt.AnswerStatus, "clarify", StringComparison.OrdinalIgnoreCase)); + var insufficientAnswers = signals.Count(static evt => + evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) && + string.Equals(evt.AnswerStatus, "insufficient", StringComparison.OrdinalIgnoreCase)); + + metrics.FallbackAnswerRate = totalAnswerFrames == 0 + ? 0d + : Math.Round((double)fallbackAnswers / totalAnswerFrames * 100d, 1); + metrics.ClarifyRate = totalAnswerFrames == 0 + ? 0d + : Math.Round((double)clarifyAnswers / totalAnswerFrames * 100d, 1); + metrics.InsufficientRate = totalAnswerFrames == 0 + ? 0d + : Math.Round((double)insufficientAnswers / totalAnswerFrames * 100d, 1); + metrics.ReformulationCount = signals.Count(static evt => + evt.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase)); + metrics.RescueActionCount = signals.Count(static evt => + evt.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase)); + metrics.AbandonedFallbackCount = GetAbandonedFallbackSessions(signals).Count; + } + + private static IReadOnlyList BuildFallbackLoopCandidates(IReadOnlyList events) + { + var rawCandidates = new List(); + foreach (var group in events + .Where(static evt => !string.IsNullOrWhiteSpace(evt.SessionId)) + .GroupBy(static evt => evt.SessionId!, StringComparer.Ordinal)) + { + var ordered = OrderSessionEvents(group); + var fallbackFrames = ordered.Where(IsFallbackAnswerEvent).ToArray(); + if (fallbackFrames.Length < FallbackLoopAlertThreshold) + { + continue; + } + + var first = fallbackFrames[0]; + var last = fallbackFrames[^1]; + rawCandidates.Add(new AlertCandidate(last.Query, fallbackFrames.Length, first.CreatedAt, last.CreatedAt)); + } + + return rawCandidates + .GroupBy(static candidate => candidate.Query, StringComparer.OrdinalIgnoreCase) + .Select(group => new AlertCandidate( + group.Key, + group.Sum(static candidate => candidate.OccurrenceCount), + group.Min(static candidate => candidate.FirstSeen), + group.Max(static candidate => candidate.LastSeen))) + .OrderByDescending(static candidate => candidate.OccurrenceCount) + .ToArray(); + } + + private static IReadOnlyList BuildAbandonedFallbackCandidates(IReadOnlyList events) + { + return GetAbandonedFallbackSessions(events) + .GroupBy(static candidate => candidate.Query, StringComparer.OrdinalIgnoreCase) + .Select(group => new AlertCandidate( + group.Key, + group.Sum(static candidate => candidate.OccurrenceCount), + group.Min(static candidate => candidate.FirstSeen), + group.Max(static candidate => candidate.LastSeen))) + .Where(static candidate => candidate.OccurrenceCount >= AbandonedFallbackAlertThreshold) + .OrderByDescending(static candidate => candidate.OccurrenceCount) + .ToArray(); + } + + private static IReadOnlyList GetAbandonedFallbackSessions(IReadOnlyList events) + { + var candidates = new List(); + foreach (var group in events + .Where(static evt => !string.IsNullOrWhiteSpace(evt.SessionId)) + .GroupBy(static evt => evt.SessionId!, StringComparer.Ordinal)) + { + var ordered = OrderSessionEvents(group); + var lastFallbackIndex = -1; + for (var index = ordered.Length - 1; index >= 0; index--) + { + if (IsFallbackAnswerEvent(ordered[index])) + { + lastFallbackIndex = index; + break; + } + } + + if (lastFallbackIndex < 0) + { + continue; + } + + var lastFallback = ordered[lastFallbackIndex]; + var recovered = false; + for (var index = lastFallbackIndex + 1; index < ordered.Length; index++) + { + if (IsRecoveryEvent(ordered[index])) + { + recovered = true; + break; + } + } + + if (recovered) + { + continue; + } + + candidates.Add(new AlertCandidate(lastFallback.Query, 1, lastFallback.CreatedAt, lastFallback.CreatedAt)); + } + + return candidates; + } + + private static bool IsFallbackAnswerEvent(SelfServeSignalEvent evt) + { + return evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) + && (string.Equals(evt.AnswerStatus, "clarify", StringComparison.OrdinalIgnoreCase) + || string.Equals(evt.AnswerStatus, "insufficient", StringComparison.OrdinalIgnoreCase)); + } + + private static bool IsRecoveryEvent(SelfServeSignalEvent evt) + { + return evt.EventType.Equals("click", StringComparison.OrdinalIgnoreCase) + || evt.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase) + || evt.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase) + || (evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) + && string.Equals(evt.AnswerStatus, "grounded", StringComparison.OrdinalIgnoreCase)); + } + + private static SelfServeSignalEvent[] OrderSessionEvents(IEnumerable events) + { + return events + .OrderBy(static evt => evt.CreatedAt) + .ThenBy(static evt => evt.OrderKey, StringComparer.Ordinal) + .ToArray(); + } + + private static string? NullIfEmpty(string value) + { + return string.IsNullOrWhiteSpace(value) ? null : value; + } + private async Task> LoadZeroResultCandidatesAsync( string tenantId, TimeSpan window, @@ -1170,12 +1434,26 @@ internal sealed class SearchQualityMetricsEntry public double ZeroResultRate { get; set; } public double AvgResultCount { get; set; } public double FeedbackScore { get; set; } + public double FallbackAnswerRate { get; set; } + public double ClarifyRate { get; set; } + public double InsufficientRate { get; set; } + public int ReformulationCount { get; set; } + public int RescueActionCount { get; set; } + public int AbandonedFallbackCount { get; set; } public string Period { get; set; } = "7d"; public IReadOnlyList LowQualityResults { get; set; } = []; public IReadOnlyList TopQueries { get; set; } = []; public IReadOnlyList Trend { get; set; } = []; } +internal sealed record SelfServeSignalEvent( + string? SessionId, + string EventType, + string Query, + string? AnswerStatus, + DateTimeOffset CreatedAt, + string OrderKey); + internal sealed class SearchQualityLowQualityRow { public string EntityKey { get; set; } = string.Empty; diff --git a/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/UnifiedSearchService.cs b/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/UnifiedSearchService.cs index e75672c94..6e8f84da4 100644 --- a/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/UnifiedSearchService.cs +++ b/src/AdvisoryAI/StellaOps.AdvisoryAI/UnifiedSearch/UnifiedSearchService.cs @@ -107,18 +107,23 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService return EmptyResponse(string.Empty, request.K, "empty"); } - if (query.Length > _unifiedOptions.MaxQueryLength) - { - return EmptyResponse(query, request.K, "query_too_long", request.Ambient); - } - var tenantId = request.Filters?.Tenant ?? "global"; var userId = request.Filters?.UserId ?? "anonymous"; + + if (query.Length > _unifiedOptions.MaxQueryLength) + { + var earlyResponse = EmptyResponse(query, request.K, "query_too_long", request.Ambient); + await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false); + return earlyResponse; + } + var tenantFlags = ResolveTenantFeatureFlags(tenantId); if (!_options.Enabled || !IsSearchEnabledForTenant(tenantFlags) || string.IsNullOrWhiteSpace(_options.ConnectionString)) { - return EmptyResponse(query, request.K, "disabled", request.Ambient); + var earlyResponse = EmptyResponse(query, request.K, "disabled", request.Ambient); + await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false); + return earlyResponse; } if (request.Ambient?.ResetSession == true && @@ -321,6 +326,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService refinements, contextAnswer); + await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan, response, cancellationToken).ConfigureAwait(false); EmitTelemetry(plan, response, tenantId); return response; } @@ -1858,4 +1864,40 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService HasSuggestions: response.Suggestions is { Count: > 0 }, HasRefinements: response.Refinements is { Count: > 0 })); } + + private async Task RecordAnswerFrameAnalyticsAsync( + string tenantId, + string userId, + UnifiedSearchRequest request, + QueryPlan? plan, + UnifiedSearchResponse response, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(response.Query)) + { + return; + } + + var contextAnswer = response.ContextAnswer; + if (contextAnswer is null) + { + return; + } + + var domain = ResolveContextDomain(plan, response.Cards, request.Ambient) + ?? response.Cards.FirstOrDefault()?.Domain + ?? request.Filters?.Domains?.FirstOrDefault(); + + await _analyticsService.RecordEventAsync(new SearchAnalyticsEvent( + TenantId: tenantId, + EventType: "answer_frame", + Query: response.Query, + UserId: userId, + SessionId: request.Ambient?.SessionId, + Domain: domain, + ResultCount: response.Cards.Count, + DurationMs: (int)Math.Min(int.MaxValue, Math.Max(0, response.Diagnostics.DurationMs)), + AnswerStatus: contextAnswer.Status, + AnswerCode: contextAnswer.Code), cancellationToken).ConfigureAwait(false); + } } diff --git a/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/Integration/UnifiedSearchSprintIntegrationTests.cs b/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/Integration/UnifiedSearchSprintIntegrationTests.cs index a4d33cdf8..e86218558 100644 --- a/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/Integration/UnifiedSearchSprintIntegrationTests.cs +++ b/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/Integration/UnifiedSearchSprintIntegrationTests.cs @@ -1511,6 +1511,12 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable "zero-result rate must be computed from raw zero_result analytics events"); metrics.AvgResultCount.Should().BeApproximately(2.0, 0.2); metrics.FeedbackScore.Should().BeApproximately(50.0, 0.2); + metrics.FallbackAnswerRate.Should().Be(0); + metrics.ClarifyRate.Should().Be(0); + metrics.InsufficientRate.Should().Be(0); + metrics.ReformulationCount.Should().Be(0); + metrics.RescueActionCount.Should().Be(0); + metrics.AbandonedFallbackCount.Should().Be(0); } [Fact] @@ -1579,6 +1585,74 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable .Should().BeTrue("low-quality rows should include repeated negative feedback entities"); } + [Fact] + public async Task G10_SelfServeMetrics_IncludeFallbackReformulationAndRescueSignals() + { + var tenant = $"selfserve-metrics-{Guid.NewGuid():N}"; + using var writer = CreateAuthenticatedClient(tenant); + using var admin = CreateAdminClient(tenant); + + var analyticsResponse = await writer.PostAsJsonAsync("/v1/advisory-ai/search/analytics", new SearchAnalyticsApiRequest + { + Events = + [ + new SearchAnalyticsApiEvent + { + EventType = "query", + Query = "database connectivity", + ResultCount = 0 + }, + new SearchAnalyticsApiEvent + { + EventType = "answer_frame", + Query = "database connectivity", + SessionId = "session-selfserve-1", + Domain = "knowledge", + AnswerStatus = "clarify", + AnswerCode = "query_needs_scope" + }, + new SearchAnalyticsApiEvent + { + EventType = "reformulation", + Query = "postgresql connectivity", + SessionId = "session-selfserve-1", + Domain = "knowledge" + }, + new SearchAnalyticsApiEvent + { + EventType = "answer_frame", + Query = "postgresql connectivity", + SessionId = "session-selfserve-1", + Domain = "knowledge", + AnswerStatus = "insufficient", + AnswerCode = "no_grounded_evidence" + }, + new SearchAnalyticsApiEvent + { + EventType = "rescue_action", + Query = "postgresql connectivity", + SessionId = "session-selfserve-1", + Domain = "knowledge" + } + ] + }); + + analyticsResponse.StatusCode.Should().Be(HttpStatusCode.NoContent); + + var metricsResponse = await admin.GetAsync("/v1/advisory-ai/search/quality/metrics?period=7d"); + metricsResponse.StatusCode.Should().Be(HttpStatusCode.OK); + + var metrics = await metricsResponse.Content.ReadFromJsonAsync(); + metrics.Should().NotBeNull(); + metrics!.FallbackAnswerRate.Should().BeApproximately(100.0, 0.2); + metrics.ClarifyRate.Should().BeApproximately(50.0, 0.2); + metrics.InsufficientRate.Should().BeApproximately(50.0, 0.2); + metrics.ReformulationCount.Should().Be(1); + metrics.RescueActionCount.Should().Be(1); + metrics.AbandonedFallbackCount.Should().Be(0, + "a rescue action in the same session means the fallback flow was not abandoned"); + } + [Fact] public async Task G10_AnalyticsEndpoint_ValidBatch_ReturnsNoContent() { @@ -1624,6 +1698,7 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable EventType: "query", Query: query, UserId: userId, + SessionId: "session-privacy-1", ResultCount: 1)); var events = analyticsService.GetFallbackEventsSnapshot(tenant, TimeSpan.FromMinutes(1)); @@ -1634,6 +1709,7 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable stored.Query.Should().NotContain("alice", "raw query text must not be stored in analytics events"); stored.UserId.Should().Be(SearchAnalyticsPrivacy.HashUserId(tenant, userId)); stored.UserId.Should().NotBe(userId, "user identifiers must be pseudonymized before persistence"); + stored.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId(tenant, "session-privacy-1")); } [Fact] @@ -1862,6 +1938,76 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable .Should().BeTrue("five repeated zero-result events should create a zero_result quality alert"); } + [Fact] + public async Task G10_FallbackLoopSessions_CreateQualityAlert() + { + var tenant = $"fallback-loop-{Guid.NewGuid():N}"; + using var scope = _factory.Services.CreateScope(); + var analytics = scope.ServiceProvider.GetRequiredService(); + var monitor = scope.ServiceProvider.GetRequiredService(); + + await analytics.RecordEventsAsync( + [ + new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate", SessionId: "sess-loop-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"), + new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate release", SessionId: "sess-loop-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"), + new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate release production", SessionId: "sess-loop-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence") + ]); + + await monitor.RefreshAlertsAsync(tenant); + var alerts = await monitor.GetAlertsAsync(tenant, status: "open", alertType: "fallback_loop"); + + var expectedQueryHash = SearchAnalyticsPrivacy.HashQuery("policy gate release production"); + alerts.Any(alert => alert.Query.Equals(expectedQueryHash, StringComparison.OrdinalIgnoreCase)) + .Should().BeTrue("three fallback answer frames in the same session should create a fallback_loop alert"); + } + + [Fact] + public async Task G10_AbandonedFallbackSessions_CreateQualityAlert() + { + var tenant = $"abandoned-fallback-{Guid.NewGuid():N}"; + using var scope = _factory.Services.CreateScope(); + var analytics = scope.ServiceProvider.GetRequiredService(); + var monitor = scope.ServiceProvider.GetRequiredService(); + + await analytics.RecordEventsAsync( + [ + new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"), + new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-2", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"), + new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-3", AnswerStatus: "clarify", AnswerCode: "query_needs_scope") + ]); + + await monitor.RefreshAlertsAsync(tenant); + var alerts = await monitor.GetAlertsAsync(tenant, status: "open", alertType: "abandoned_fallback"); + + var expectedQueryHash = SearchAnalyticsPrivacy.HashQuery("unknown control token"); + alerts.Any(alert => alert.Query.Equals(expectedQueryHash, StringComparison.OrdinalIgnoreCase)) + .Should().BeTrue("repeated sessions that end in fallback without recovery should create an abandoned_fallback alert"); + } + + [Fact] + public async Task G10_RecoveredFallbackSessions_DoNotCountAsAbandoned() + { + var tenant = $"recovered-fallback-{Guid.NewGuid():N}"; + using var scope = _factory.Services.CreateScope(); + var analytics = scope.ServiceProvider.GetRequiredService(); + var monitor = scope.ServiceProvider.GetRequiredService(); + + await analytics.RecordEventsAsync( + [ + new SearchAnalyticsEvent(tenant, "answer_frame", "database connectivity", SessionId: "sess-recovered-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"), + new SearchAnalyticsEvent(tenant, "reformulation", "postgresql connectivity", SessionId: "sess-recovered-1"), + new SearchAnalyticsEvent(tenant, "answer_frame", "postgresql connectivity", SessionId: "sess-recovered-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"), + new SearchAnalyticsEvent(tenant, "rescue_action", "postgresql connectivity", SessionId: "sess-recovered-1") + ]); + + var metrics = await monitor.GetMetricsAsync(tenant, "7d"); + + metrics.ReformulationCount.Should().Be(1); + metrics.RescueActionCount.Should().Be(1); + metrics.AbandonedFallbackCount.Should().Be(0, + "a session with a reformulation and rescue action after fallback is a recovery path, not an abandonment"); + } + [Fact] public async Task G10_NegativeFeedbackBurst_CreatesHighNegativeFeedbackAlert() { diff --git a/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/TASKS.md b/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/TASKS.md index 8fd01c575..a4798ce70 100644 --- a/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/TASKS.md +++ b/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/TASKS.md @@ -22,4 +22,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol | SPRINT_20260224_G1-004-BENCH | DONE | Semantic recall benchmark: 13 tests in `SemanticRecallBenchmarkTests.cs`, 48-query fixture (`semantic-recall-benchmark.json`), `SemanticRecallBenchmarkStore` (33 chunks), `SemanticSimulationEncoder` (40+ semantic groups). Semantic strictly outperforms hash on synonym queries. | | AI-SELF-005 | DONE | Integration coverage now asserts grounded, clarify, and insufficient contextual-answer states through the real endpoint contract. | +| AI-SELF-004 | DONE | Targeted telemetry coverage now verifies fallback metrics, recovered-session suppression of abandoned counts, session hashing, and self-serve alert generation with xUnit v3-compatible runner commands. | | AI-SELF-006 | DONE | Verification includes a real local corpus rebuild and a live query assertion, not only test doubles. | diff --git a/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/UnifiedSearch/UnifiedSearchServiceTests.cs b/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/UnifiedSearch/UnifiedSearchServiceTests.cs index 6bc3046c2..b8f455c9d 100644 --- a/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/UnifiedSearch/UnifiedSearchServiceTests.cs +++ b/src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/UnifiedSearch/UnifiedSearchServiceTests.cs @@ -8,6 +8,7 @@ using StellaOps.AdvisoryAI.UnifiedSearch.Analytics; using StellaOps.AdvisoryAI.UnifiedSearch.QueryUnderstanding; using StellaOps.AdvisoryAI.UnifiedSearch.Synthesis; using StellaOps.AdvisoryAI.Vectorization; +using System.Linq; using System.Text.Json; using Xunit; @@ -124,6 +125,101 @@ public sealed class UnifiedSearchServiceTests result.ContextAnswer.Questions!.Should().OnlyContain(question => question.Kind == "recover"); } + [Fact] + public async Task SearchAsync_records_answer_frame_analytics_for_clarify_state() + { + var analyticsOptions = Options.Create(new KnowledgeSearchOptions + { + Enabled = true, + ConnectionString = "Host=localhost;Database=test" + }); + var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger.Instance); + var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger.Instance, analyticsService); + var service = CreateService( + storeMock: CreateEmptyStoreMock(), + analyticsService: analyticsService, + qualityMonitor: qualityMonitor); + + var result = await service.SearchAsync( + new UnifiedSearchRequest( + "status", + Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" }, + Ambient: new AmbientContext + { + CurrentRoute = "/ops/operations/doctor", + SessionId = "session-123" + }), + CancellationToken.None); + + result.ContextAnswer.Should().NotBeNull(); + result.ContextAnswer!.Status.Should().Be("clarify"); + + var events = analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1)); + events.Should().ContainSingle(); + var answerFrame = events.Single().Event; + answerFrame.EventType.Should().Be("answer_frame"); + answerFrame.AnswerStatus.Should().Be("clarify"); + answerFrame.AnswerCode.Should().Be(result.ContextAnswer.Code); + answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-123")); + answerFrame.Query.Should().Be(SearchAnalyticsPrivacy.HashQuery("status")); + } + + [Fact] + public async Task SearchAsync_records_answer_frame_analytics_for_grounded_state() + { + var analyticsOptions = Options.Create(new KnowledgeSearchOptions + { + Enabled = true, + ConnectionString = "Host=localhost;Database=test" + }); + var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger.Instance); + var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger.Instance, analyticsService); + + var doctorRow = MakeRow( + "chunk-doctor-analytics", + "doctor_check", + "PostgreSQL connectivity", + JsonDocument.Parse("{\"domain\":\"knowledge\",\"checkCode\":\"check.core.db.connectivity\"}")); + + var storeMock = new Mock(); + storeMock.Setup(s => s.SearchFtsAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new List { doctorRow }); + storeMock.Setup(s => s.LoadVectorCandidatesAsync( + It.IsAny(), It.IsAny(), It.IsAny(), + It.IsAny(), It.IsAny())) + .ReturnsAsync([]); + + var service = CreateService( + storeMock: storeMock, + analyticsService: analyticsService, + qualityMonitor: qualityMonitor); + + var result = await service.SearchAsync( + new UnifiedSearchRequest( + "database connectivity", + Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" }, + Ambient: new AmbientContext + { + CurrentRoute = "/ops/operations/doctor", + SessionId = "session-456" + }), + CancellationToken.None); + + result.ContextAnswer.Should().NotBeNull(); + result.ContextAnswer!.Status.Should().Be("grounded"); + + var events = analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1)); + events.Should().ContainSingle(); + var answerFrame = events.Single().Event; + answerFrame.EventType.Should().Be("answer_frame"); + answerFrame.AnswerStatus.Should().Be("grounded"); + answerFrame.AnswerCode.Should().Be("retrieved_evidence"); + answerFrame.ResultCount.Should().Be(1); + answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-456")); + } + [Fact] public async Task SearchAsync_returns_empty_when_tenant_feature_flag_disables_search() { @@ -827,7 +923,9 @@ public sealed class UnifiedSearchServiceTests private static UnifiedSearchService CreateService( bool enabled = true, Mock? storeMock = null, - UnifiedSearchOptions? unifiedOptions = null) + UnifiedSearchOptions? unifiedOptions = null, + SearchAnalyticsService? analyticsService = null, + SearchQualityMonitor? qualityMonitor = null) { var options = Options.Create(new KnowledgeSearchOptions { @@ -855,8 +953,8 @@ public sealed class UnifiedSearchServiceTests var weightCalculator = new DomainWeightCalculator(extractor, classifier, options); var planBuilder = new QueryPlanBuilder(extractor, classifier, weightCalculator); var synthesisEngine = new SynthesisTemplateEngine(); - var analyticsService = new SearchAnalyticsService(options, NullLogger.Instance); - var qualityMonitor = new SearchQualityMonitor(options, NullLogger.Instance); + analyticsService ??= new SearchAnalyticsService(options, NullLogger.Instance); + qualityMonitor ??= new SearchQualityMonitor(options, NullLogger.Instance, analyticsService); var entityAliasService = new Mock(); entityAliasService.Setup(s => s.ResolveAliasesAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(Array.Empty<(string EntityKey, string EntityType)>());