Add self-serve search telemetry gap surfacing
This commit is contained in:
@@ -65,7 +65,7 @@ Completion criteria:
|
||||
- [ ] Ambiguous queries return clarifying prompts instead of a blank answer slot.
|
||||
|
||||
### AI-SELF-004 - Self-serve telemetry and gap surfacing
|
||||
Status: TODO
|
||||
Status: DONE
|
||||
Dependency: AI-SELF-002
|
||||
Owners: Developer (AdvisoryAI), Test Automation
|
||||
Task description:
|
||||
@@ -73,9 +73,9 @@ Task description:
|
||||
- Expose enough structured data to drive a gap-closure backlog.
|
||||
|
||||
Completion criteria:
|
||||
- [ ] Telemetry captures unanswered and reformulated journeys without persisting raw sensitive prompts unnecessarily.
|
||||
- [ ] Operational docs explain how to review self-serve gaps.
|
||||
- [ ] Tests cover telemetry emission for fallback paths.
|
||||
- [x] Telemetry captures unanswered and reformulated journeys without persisting raw sensitive prompts unnecessarily.
|
||||
- [x] Operational docs explain how to review self-serve gaps.
|
||||
- [x] Tests cover telemetry emission for fallback paths.
|
||||
|
||||
### AI-SELF-005 - Targeted behavioral verification
|
||||
Status: DONE
|
||||
@@ -110,6 +110,8 @@ Completion criteria:
|
||||
| 2026-03-07 | Implemented `contextAnswer` in unified search/backend API mapping, added deterministic `grounded` / `clarify` / `insufficient` rules plus follow-up question generation, and extended telemetry fields for answer-state visibility. | Developer |
|
||||
| 2026-03-07 | Verified the AdvisoryAI test project after the contract change with `dotnet test "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj" --no-restore -v normal` (`877/877` passing). | Test Automation |
|
||||
| 2026-03-07 | Exercised the live rebuilt-corpus lane against `http://127.0.0.1:10451`: `POST /v1/advisory-ai/index/rebuild`, `POST /v1/search/index/rebuild`, then `POST /v1/search/query` for `database connectivity`, which returned `contextAnswer.status = grounded`, 3 citations, and 10 cards over ingested data. | Test Automation |
|
||||
| 2026-03-07 | Implemented optional self-serve analytics columns (`session_id`, `answer_status`, `answer_code`), metrics (`fallbackAnswerRate`, `clarifyRate`, `insufficientRate`, `reformulationCount`, `rescueActionCount`, `abandonedFallbackCount`), and alerting (`fallback_loop`, `abandoned_fallback`) with privacy-preserving session hashing. | Developer |
|
||||
| 2026-03-07 | Verified targeted telemetry coverage with xUnit v3 runner commands against `UnifiedSearchSprintIntegrationTests` and added a recovered-session regression test so rescue actions clear abandonment as expected. | Test Automation |
|
||||
|
||||
## Decisions & Risks
|
||||
- Decision: the backend contract must return explicit answer states instead of leaving the UI to infer confidence from cards alone.
|
||||
@@ -122,6 +124,8 @@ Completion criteria:
|
||||
- Risk: mocked endpoint tests can overstate confidence if ingestion adapters or corpus rebuild order drift.
|
||||
- Mitigation: keep rebuild order documented, execute it during verification, and record which routes have live-ingested parity.
|
||||
- Decision: `stella advisoryai sources prepare` is optional for local verification when checked-in Doctor seed/control files are already sufficient, but it requires `STELLAOPS_BACKEND_URL` whenever live Doctor discovery is expected.
|
||||
- Decision: self-serve analytics is optional and additive; search behavior must not depend on telemetry event emission.
|
||||
- Decision: targeted AdvisoryAI verification uses xUnit v3 / Microsoft.Testing.Platform-compatible filters (`-- --filter-class` or the built test executable), not legacy VSTest `--filter`.
|
||||
|
||||
## Next Checkpoints
|
||||
- 2026-03-10: Freeze answer payload shape and fallback taxonomy.
|
||||
|
||||
@@ -17,7 +17,10 @@ public static class SearchAnalyticsEndpoints
|
||||
"query",
|
||||
"click",
|
||||
"zero_result",
|
||||
"synthesis"
|
||||
"synthesis",
|
||||
"answer_frame",
|
||||
"reformulation",
|
||||
"rescue_action"
|
||||
};
|
||||
|
||||
public static RouteGroupBuilder MapSearchAnalyticsEndpoints(this IEndpointRouteBuilder builder)
|
||||
@@ -30,10 +33,11 @@ public static class SearchAnalyticsEndpoints
|
||||
|
||||
group.MapPost("/analytics", RecordAnalyticsAsync)
|
||||
.WithName("SearchAnalyticsRecord")
|
||||
.WithSummary("Records batch search analytics events (query, click, zero_result, synthesis).")
|
||||
.WithSummary("Records batch search analytics events (query, click, zero_result, synthesis, answer_frame, reformulation, rescue_action).")
|
||||
.WithDescription(
|
||||
"Accepts a batch of search analytics events for tracking query frequency, click-through rates, " +
|
||||
"zero-result queries, and synthesis usage. Queries and user identifiers are pseudonymized before persistence. " +
|
||||
"zero-result queries, self-serve answer states, reformulations, rescue-action usage, and synthesis usage. " +
|
||||
"Queries and user/session identifiers are pseudonymized before persistence. " +
|
||||
"Fire-and-forget from the client; failures do not affect search functionality.")
|
||||
.RequireAuthorization(AdvisoryAIPolicies.OperatePolicy)
|
||||
.Produces(StatusCodes.Status204NoContent)
|
||||
@@ -113,11 +117,14 @@ public static class SearchAnalyticsEndpoints
|
||||
EventType: apiEvent.EventType.Trim().ToLowerInvariant(),
|
||||
Query: apiEvent.Query.Trim(),
|
||||
UserId: userId,
|
||||
SessionId: string.IsNullOrWhiteSpace(apiEvent.SessionId) ? null : apiEvent.SessionId.Trim(),
|
||||
EntityKey: string.IsNullOrWhiteSpace(apiEvent.EntityKey) ? null : apiEvent.EntityKey.Trim(),
|
||||
Domain: string.IsNullOrWhiteSpace(apiEvent.Domain) ? null : apiEvent.Domain.Trim(),
|
||||
ResultCount: apiEvent.ResultCount,
|
||||
Position: apiEvent.Position,
|
||||
DurationMs: apiEvent.DurationMs));
|
||||
DurationMs: apiEvent.DurationMs,
|
||||
AnswerStatus: string.IsNullOrWhiteSpace(apiEvent.AnswerStatus) ? null : apiEvent.AnswerStatus.Trim(),
|
||||
AnswerCode: string.IsNullOrWhiteSpace(apiEvent.AnswerCode) ? null : apiEvent.AnswerCode.Trim()));
|
||||
}
|
||||
|
||||
if (events.Count > 0)
|
||||
@@ -301,6 +308,8 @@ public sealed record SearchAnalyticsApiEvent
|
||||
|
||||
public string Query { get; init; } = string.Empty;
|
||||
|
||||
public string? SessionId { get; init; }
|
||||
|
||||
public string? EntityKey { get; init; }
|
||||
|
||||
public string? Domain { get; init; }
|
||||
@@ -310,6 +319,10 @@ public sealed record SearchAnalyticsApiEvent
|
||||
public int? Position { get; init; }
|
||||
|
||||
public int? DurationMs { get; init; }
|
||||
|
||||
public string? AnswerStatus { get; init; }
|
||||
|
||||
public string? AnswerCode { get; init; }
|
||||
}
|
||||
|
||||
public sealed record SearchHistoryApiResponse
|
||||
|
||||
@@ -39,11 +39,11 @@ public static class SearchFeedbackEndpoints
|
||||
// G10-002: List quality alerts (admin only)
|
||||
group.MapGet("/quality/alerts", GetAlertsAsync)
|
||||
.WithName("SearchQualityAlertsList")
|
||||
.WithSummary("Lists open search quality alerts (zero-result queries, high negative feedback).")
|
||||
.WithSummary("Lists open search quality alerts (zero-result, high negative feedback, fallback loops, abandoned fallback).")
|
||||
.WithDescription(
|
||||
"Returns search quality alerts ordered by occurrence count. " +
|
||||
"Filterable by status (open, acknowledged, resolved) and alert type " +
|
||||
"(zero_result, low_feedback, high_negative_feedback). Requires admin scope.")
|
||||
"(zero_result, high_negative_feedback, fallback_loop, abandoned_fallback). Requires admin scope.")
|
||||
.RequireAuthorization(AdvisoryAIPolicies.AdminPolicy)
|
||||
.Produces<IReadOnlyList<SearchQualityAlertDto>>(StatusCodes.Status200OK)
|
||||
.Produces(StatusCodes.Status403Forbidden);
|
||||
@@ -198,6 +198,12 @@ public static class SearchFeedbackEndpoints
|
||||
ZeroResultRate = metrics.ZeroResultRate,
|
||||
AvgResultCount = metrics.AvgResultCount,
|
||||
FeedbackScore = metrics.FeedbackScore,
|
||||
FallbackAnswerRate = metrics.FallbackAnswerRate,
|
||||
ClarifyRate = metrics.ClarifyRate,
|
||||
InsufficientRate = metrics.InsufficientRate,
|
||||
ReformulationCount = metrics.ReformulationCount,
|
||||
RescueActionCount = metrics.RescueActionCount,
|
||||
AbandonedFallbackCount = metrics.AbandonedFallbackCount,
|
||||
Period = metrics.Period,
|
||||
LowQualityResults = metrics.LowQualityResults
|
||||
.Select(row => new SearchLowQualityResultDto
|
||||
@@ -308,6 +314,12 @@ public sealed record SearchQualityMetricsDto
|
||||
public double ZeroResultRate { get; init; }
|
||||
public double AvgResultCount { get; init; }
|
||||
public double FeedbackScore { get; init; }
|
||||
public double FallbackAnswerRate { get; init; }
|
||||
public double ClarifyRate { get; init; }
|
||||
public double InsufficientRate { get; init; }
|
||||
public int ReformulationCount { get; init; }
|
||||
public int RescueActionCount { get; init; }
|
||||
public int AbandonedFallbackCount { get; init; }
|
||||
public string Period { get; init; } = "7d";
|
||||
public IReadOnlyList<SearchLowQualityResultDto> LowQualityResults { get; init; } = [];
|
||||
public IReadOnlyList<SearchTopQueryDto> TopQueries { get; init; } = [];
|
||||
|
||||
@@ -17,4 +17,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
|
||||
| SPRINT_20260224_G1-G10 | DONE | Search improvement sprints G1–G10 implemented. New endpoints: `SearchAnalyticsEndpoints.cs` (history, events, popularity), `SearchFeedbackEndpoints.cs` (feedback, quality alerts, metrics). Extended: `UnifiedSearchEndpoints.cs` (suggestions, refinements, previews, diagnostics.activeEncoder). Extended: `KnowledgeSearchEndpoints.cs` (activeEncoder in diagnostics). See `docs/modules/advisory-ai/knowledge-search.md` for full testing guide. |
|
||||
|
||||
| AI-SELF-001 | DONE | Unified search endpoint contract now exposes backend contextual answer fields for self-serve search. |
|
||||
| AI-SELF-004 | DONE | Search analytics and quality endpoints now surface self-serve metrics and alerts (`fallback_loop`, `abandoned_fallback`) while keeping telemetry optional. |
|
||||
| AI-SELF-006 | DONE | Endpoint readiness now includes a proven local rebuilt-corpus verification lane in addition to stubbed integration tests. |
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
-- 008_search_self_serve_analytics.sql
|
||||
-- Adds self-serve telemetry fields to search_events so fallback loops,
|
||||
-- reformulations, rescue actions, and abandoned fallback sessions can be tracked.
|
||||
|
||||
ALTER TABLE advisoryai.search_events
|
||||
ADD COLUMN IF NOT EXISTS session_id TEXT;
|
||||
|
||||
ALTER TABLE advisoryai.search_events
|
||||
ADD COLUMN IF NOT EXISTS answer_status TEXT;
|
||||
|
||||
ALTER TABLE advisoryai.search_events
|
||||
ADD COLUMN IF NOT EXISTS answer_code TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_search_events_tenant_session
|
||||
ON advisoryai.search_events (tenant_id, session_id, created_at)
|
||||
WHERE session_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_search_events_answer_status
|
||||
ON advisoryai.search_events (tenant_id, answer_status, created_at)
|
||||
WHERE answer_status IS NOT NULL;
|
||||
@@ -13,7 +13,7 @@ Source of truth: `docs/implplan/SPRINT_20260113_005_ADVISORYAI_controlled_conver
|
||||
| AI-SELF-001 | DONE | Unified search now emits the contextual answer payload (`contextAnswer`) with answer state, citations, and follow-up questions for self-serve search. |
|
||||
| AI-SELF-002 | DONE | Deterministic grounded/clarify/insufficient fallback policy is implemented in unified search orchestration. |
|
||||
| AI-SELF-003 | DONE | Follow-up question generation from route/domain intent, recent actions, and evidence is implemented. |
|
||||
| AI-SELF-004 | TODO | Telemetry for unanswered and reformulated self-serve journeys is still pending. |
|
||||
| AI-SELF-004 | DONE | Optional self-serve telemetry now captures answer frames, reformulations, rescue actions, and abandonment signals via hashed session ids; quality metrics and alerts expose the gaps operationally. |
|
||||
| AI-SELF-006 | DONE | Live ingestion-backed answer verification succeeded on the Doctor/knowledge route after local rebuild. |
|
||||
| SPRINT_20260222_051-AKS-INGEST | DONE | Added deterministic AKS ingestion controls: markdown allow-list manifest loading, OpenAPI aggregate source path support, and doctor control projection integration for search chunks, including fallback doctor metadata hydration from controls projection fields. |
|
||||
| AUDIT-0017-M | DONE | Maintainability audit for StellaOps.AdvisoryAI. |
|
||||
|
||||
@@ -32,6 +32,19 @@ internal static class SearchAnalyticsPrivacy
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
public static string? HashSessionId(string tenantId, string? sessionId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sessionId))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalizedTenant = tenantId.Trim().ToLowerInvariant();
|
||||
var normalizedSession = sessionId.Trim().ToLowerInvariant();
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes($"{normalizedTenant}|session|{normalizedSession}"));
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
public static string? RedactFreeform(string? value)
|
||||
{
|
||||
_ = value;
|
||||
|
||||
@@ -37,11 +37,14 @@ internal sealed class SearchAnalyticsService
|
||||
await conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
|
||||
await using var cmd = new NpgsqlCommand(@"
|
||||
INSERT INTO advisoryai.search_events (tenant_id, user_id, event_type, query, entity_key, domain, result_count, position, duration_ms)
|
||||
VALUES (@tenant_id, @user_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms)", conn);
|
||||
INSERT INTO advisoryai.search_events
|
||||
(tenant_id, user_id, session_id, event_type, query, entity_key, domain, result_count, position, duration_ms, answer_status, answer_code)
|
||||
VALUES
|
||||
(@tenant_id, @user_id, @session_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms, @answer_status, @answer_code)", conn);
|
||||
|
||||
cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId);
|
||||
cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("session_id", (object?)persistedEvent.SessionId ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType);
|
||||
cmd.Parameters.AddWithValue("query", persistedEvent.Query);
|
||||
cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value);
|
||||
@@ -49,6 +52,8 @@ internal sealed class SearchAnalyticsService
|
||||
cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("answer_status", (object?)persistedEvent.AnswerStatus ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("answer_code", (object?)persistedEvent.AnswerCode ?? DBNull.Value);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
RecordFallbackEvent(persistedEvent, recordedAt);
|
||||
@@ -88,11 +93,14 @@ internal sealed class SearchAnalyticsService
|
||||
var persistedEvent = SanitizeEvent(evt);
|
||||
|
||||
await using var cmd = new NpgsqlCommand(@"
|
||||
INSERT INTO advisoryai.search_events (tenant_id, user_id, event_type, query, entity_key, domain, result_count, position, duration_ms)
|
||||
VALUES (@tenant_id, @user_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms)", conn);
|
||||
INSERT INTO advisoryai.search_events
|
||||
(tenant_id, user_id, session_id, event_type, query, entity_key, domain, result_count, position, duration_ms, answer_status, answer_code)
|
||||
VALUES
|
||||
(@tenant_id, @user_id, @session_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms, @answer_status, @answer_code)", conn);
|
||||
|
||||
cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId);
|
||||
cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("session_id", (object?)persistedEvent.SessionId ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType);
|
||||
cmd.Parameters.AddWithValue("query", persistedEvent.Query);
|
||||
cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value);
|
||||
@@ -100,6 +108,8 @@ internal sealed class SearchAnalyticsService
|
||||
cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("answer_status", (object?)persistedEvent.AnswerStatus ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("answer_code", (object?)persistedEvent.AnswerCode ?? DBNull.Value);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
RecordFallbackEvent(persistedEvent, recordedAt);
|
||||
@@ -637,8 +647,16 @@ internal sealed class SearchAnalyticsService
|
||||
{
|
||||
return evt with
|
||||
{
|
||||
EventType = evt.EventType.Trim().ToLowerInvariant(),
|
||||
Query = SearchAnalyticsPrivacy.HashQuery(evt.Query),
|
||||
UserId = SearchAnalyticsPrivacy.HashUserId(evt.TenantId, evt.UserId)
|
||||
UserId = SearchAnalyticsPrivacy.HashUserId(evt.TenantId, evt.UserId),
|
||||
SessionId = SearchAnalyticsPrivacy.HashSessionId(evt.TenantId, evt.SessionId),
|
||||
AnswerStatus = string.IsNullOrWhiteSpace(evt.AnswerStatus)
|
||||
? null
|
||||
: evt.AnswerStatus.Trim().ToLowerInvariant(),
|
||||
AnswerCode = string.IsNullOrWhiteSpace(evt.AnswerCode)
|
||||
? null
|
||||
: evt.AnswerCode.Trim().ToLowerInvariant()
|
||||
};
|
||||
}
|
||||
|
||||
@@ -667,11 +685,14 @@ internal record SearchAnalyticsEvent(
|
||||
string EventType,
|
||||
string Query,
|
||||
string? UserId = null,
|
||||
string? SessionId = null,
|
||||
string? EntityKey = null,
|
||||
string? Domain = null,
|
||||
int? ResultCount = null,
|
||||
int? Position = null,
|
||||
int? DurationMs = null);
|
||||
int? DurationMs = null,
|
||||
string? AnswerStatus = null,
|
||||
string? AnswerCode = null);
|
||||
|
||||
internal record SearchHistoryEntry(
|
||||
string HistoryId,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Npgsql;
|
||||
@@ -6,7 +7,8 @@ using StellaOps.AdvisoryAI.KnowledgeSearch;
|
||||
namespace StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
|
||||
|
||||
/// <summary>
|
||||
/// Monitors search quality by analysing feedback data and zero-result queries.
|
||||
/// Monitors search quality by analysing feedback data, zero-result queries,
|
||||
/// and self-serve fallback telemetry.
|
||||
/// Provides CRUD for search_quality_alerts and search_feedback tables.
|
||||
/// Sprint: SPRINT_20260224_110 (G10-001, G10-002)
|
||||
/// </summary>
|
||||
@@ -17,6 +19,8 @@ internal sealed class SearchQualityMonitor
|
||||
private const int DefaultAlertWindowDays = 7;
|
||||
private const int ZeroResultAlertThreshold = 3;
|
||||
private const int NegativeFeedbackAlertThreshold = 3;
|
||||
private const int FallbackLoopAlertThreshold = 3;
|
||||
private const int AbandonedFallbackAlertThreshold = 3;
|
||||
|
||||
private readonly KnowledgeSearchOptions _options;
|
||||
private readonly ILogger<SearchQualityMonitor> _logger;
|
||||
@@ -175,6 +179,32 @@ internal sealed class SearchQualityMonitor
|
||||
candidate.LastSeen,
|
||||
ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var selfServeSignals = await LoadSelfServeSignalEventsAsync(tenantId, window, ct).ConfigureAwait(false);
|
||||
|
||||
foreach (var candidate in BuildFallbackLoopCandidates(selfServeSignals))
|
||||
{
|
||||
await UpsertAlertAsync(
|
||||
tenantId,
|
||||
alertType: "fallback_loop",
|
||||
candidate.Query,
|
||||
candidate.OccurrenceCount,
|
||||
candidate.FirstSeen,
|
||||
candidate.LastSeen,
|
||||
ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
foreach (var candidate in BuildAbandonedFallbackCandidates(selfServeSignals))
|
||||
{
|
||||
await UpsertAlertAsync(
|
||||
tenantId,
|
||||
alertType: "abandoned_fallback",
|
||||
candidate.Query,
|
||||
candidate.OccurrenceCount,
|
||||
candidate.FirstSeen,
|
||||
candidate.LastSeen,
|
||||
ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<SearchQualityAlertEntry>> GetAlertsAsync(
|
||||
@@ -423,6 +453,9 @@ internal sealed class SearchQualityMonitor
|
||||
}
|
||||
await feedbackReader.CloseAsync().ConfigureAwait(false);
|
||||
|
||||
var selfServeSignals = await LoadSelfServeSignalEventsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
||||
ApplySelfServeMetrics(metrics, selfServeSignals);
|
||||
|
||||
metrics.LowQualityResults = await LoadLowQualityResultsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
||||
metrics.TopQueries = await LoadTopQueriesAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
||||
metrics.Trend = await LoadTrendPointsAsync(conn, tenantId, ct).ConfigureAwait(false);
|
||||
@@ -473,7 +506,8 @@ internal sealed class SearchQualityMonitor
|
||||
? 0d
|
||||
: (double)helpfulCount / feedbackSignals.Length * 100d;
|
||||
|
||||
return new SearchQualityMetricsEntry
|
||||
var selfServeSignals = BuildFallbackSelfServeSignalEvents(tenantId, window);
|
||||
var metrics = new SearchQualityMetricsEntry
|
||||
{
|
||||
TotalSearches = totalSearches,
|
||||
ZeroResultRate = totalSearches == 0 ? 0d : Math.Round((double)zeroResults / totalSearches * 100d, 1),
|
||||
@@ -484,6 +518,10 @@ internal sealed class SearchQualityMonitor
|
||||
TopQueries = BuildFallbackTopQueries(tenantId, window),
|
||||
Trend = BuildFallbackTrendPoints(tenantId),
|
||||
};
|
||||
|
||||
ApplySelfServeMetrics(metrics, selfServeSignals);
|
||||
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<SearchQualityLowQualityRow>> LoadLowQualityResultsAsync(
|
||||
@@ -758,6 +796,232 @@ internal sealed class SearchQualityMonitor
|
||||
return points;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<SelfServeSignalEvent>> LoadSelfServeSignalEventsAsync(
|
||||
string tenantId,
|
||||
TimeSpan window,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var days = Math.Max(1, (int)Math.Ceiling(window.TotalDays));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(_options.ConnectionString))
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var conn = new NpgsqlConnection(_options.ConnectionString);
|
||||
await conn.OpenAsync(ct).ConfigureAwait(false);
|
||||
return await LoadSelfServeSignalEventsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Failed to load self-serve signal events from database.");
|
||||
}
|
||||
}
|
||||
|
||||
return BuildFallbackSelfServeSignalEvents(tenantId, window);
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<SelfServeSignalEvent>> LoadSelfServeSignalEventsAsync(
|
||||
NpgsqlConnection conn,
|
||||
string tenantId,
|
||||
int days,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var events = new List<SelfServeSignalEvent>();
|
||||
await using var cmd = new NpgsqlCommand(@"
|
||||
SELECT
|
||||
COALESCE(session_id, ''),
|
||||
event_type,
|
||||
query,
|
||||
answer_status,
|
||||
created_at,
|
||||
event_id::text
|
||||
FROM advisoryai.search_events
|
||||
WHERE tenant_id = @tenant_id
|
||||
AND created_at > now() - make_interval(days => @days)
|
||||
AND event_type IN ('answer_frame', 'reformulation', 'rescue_action', 'click')
|
||||
ORDER BY created_at ASC, event_id ASC", conn);
|
||||
cmd.Parameters.AddWithValue("tenant_id", tenantId);
|
||||
cmd.Parameters.AddWithValue("days", days);
|
||||
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
events.Add(new SelfServeSignalEvent(
|
||||
SessionId: reader.IsDBNull(0) ? null : NullIfEmpty(reader.GetString(0)),
|
||||
EventType: reader.GetString(1),
|
||||
Query: reader.GetString(2),
|
||||
AnswerStatus: reader.IsDBNull(3) ? null : reader.GetString(3),
|
||||
CreatedAt: new DateTimeOffset(reader.GetDateTime(4), TimeSpan.Zero),
|
||||
OrderKey: reader.GetString(5)));
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
private IReadOnlyList<SelfServeSignalEvent> BuildFallbackSelfServeSignalEvents(string tenantId, TimeSpan window)
|
||||
{
|
||||
return _analyticsService.GetFallbackEventsSnapshot(tenantId, window)
|
||||
.Where(static item =>
|
||||
item.Event.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) ||
|
||||
item.Event.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase) ||
|
||||
item.Event.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase) ||
|
||||
item.Event.EventType.Equals("click", StringComparison.OrdinalIgnoreCase))
|
||||
.Select((item, index) => new SelfServeSignalEvent(
|
||||
SessionId: item.Event.SessionId,
|
||||
EventType: item.Event.EventType,
|
||||
Query: item.Event.Query,
|
||||
AnswerStatus: item.Event.AnswerStatus,
|
||||
CreatedAt: item.RecordedAt,
|
||||
OrderKey: index.ToString("D8", CultureInfo.InvariantCulture)))
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static void ApplySelfServeMetrics(SearchQualityMetricsEntry metrics, IReadOnlyList<SelfServeSignalEvent> signals)
|
||||
{
|
||||
var totalAnswerFrames = signals.Count(static evt =>
|
||||
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase));
|
||||
var fallbackAnswers = signals.Count(IsFallbackAnswerEvent);
|
||||
var clarifyAnswers = signals.Count(static evt =>
|
||||
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(evt.AnswerStatus, "clarify", StringComparison.OrdinalIgnoreCase));
|
||||
var insufficientAnswers = signals.Count(static evt =>
|
||||
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(evt.AnswerStatus, "insufficient", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
metrics.FallbackAnswerRate = totalAnswerFrames == 0
|
||||
? 0d
|
||||
: Math.Round((double)fallbackAnswers / totalAnswerFrames * 100d, 1);
|
||||
metrics.ClarifyRate = totalAnswerFrames == 0
|
||||
? 0d
|
||||
: Math.Round((double)clarifyAnswers / totalAnswerFrames * 100d, 1);
|
||||
metrics.InsufficientRate = totalAnswerFrames == 0
|
||||
? 0d
|
||||
: Math.Round((double)insufficientAnswers / totalAnswerFrames * 100d, 1);
|
||||
metrics.ReformulationCount = signals.Count(static evt =>
|
||||
evt.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase));
|
||||
metrics.RescueActionCount = signals.Count(static evt =>
|
||||
evt.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase));
|
||||
metrics.AbandonedFallbackCount = GetAbandonedFallbackSessions(signals).Count;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AlertCandidate> BuildFallbackLoopCandidates(IReadOnlyList<SelfServeSignalEvent> events)
|
||||
{
|
||||
var rawCandidates = new List<AlertCandidate>();
|
||||
foreach (var group in events
|
||||
.Where(static evt => !string.IsNullOrWhiteSpace(evt.SessionId))
|
||||
.GroupBy(static evt => evt.SessionId!, StringComparer.Ordinal))
|
||||
{
|
||||
var ordered = OrderSessionEvents(group);
|
||||
var fallbackFrames = ordered.Where(IsFallbackAnswerEvent).ToArray();
|
||||
if (fallbackFrames.Length < FallbackLoopAlertThreshold)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var first = fallbackFrames[0];
|
||||
var last = fallbackFrames[^1];
|
||||
rawCandidates.Add(new AlertCandidate(last.Query, fallbackFrames.Length, first.CreatedAt, last.CreatedAt));
|
||||
}
|
||||
|
||||
return rawCandidates
|
||||
.GroupBy(static candidate => candidate.Query, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => new AlertCandidate(
|
||||
group.Key,
|
||||
group.Sum(static candidate => candidate.OccurrenceCount),
|
||||
group.Min(static candidate => candidate.FirstSeen),
|
||||
group.Max(static candidate => candidate.LastSeen)))
|
||||
.OrderByDescending(static candidate => candidate.OccurrenceCount)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AlertCandidate> BuildAbandonedFallbackCandidates(IReadOnlyList<SelfServeSignalEvent> events)
|
||||
{
|
||||
return GetAbandonedFallbackSessions(events)
|
||||
.GroupBy(static candidate => candidate.Query, StringComparer.OrdinalIgnoreCase)
|
||||
.Select(group => new AlertCandidate(
|
||||
group.Key,
|
||||
group.Sum(static candidate => candidate.OccurrenceCount),
|
||||
group.Min(static candidate => candidate.FirstSeen),
|
||||
group.Max(static candidate => candidate.LastSeen)))
|
||||
.Where(static candidate => candidate.OccurrenceCount >= AbandonedFallbackAlertThreshold)
|
||||
.OrderByDescending(static candidate => candidate.OccurrenceCount)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AlertCandidate> GetAbandonedFallbackSessions(IReadOnlyList<SelfServeSignalEvent> events)
|
||||
{
|
||||
var candidates = new List<AlertCandidate>();
|
||||
foreach (var group in events
|
||||
.Where(static evt => !string.IsNullOrWhiteSpace(evt.SessionId))
|
||||
.GroupBy(static evt => evt.SessionId!, StringComparer.Ordinal))
|
||||
{
|
||||
var ordered = OrderSessionEvents(group);
|
||||
var lastFallbackIndex = -1;
|
||||
for (var index = ordered.Length - 1; index >= 0; index--)
|
||||
{
|
||||
if (IsFallbackAnswerEvent(ordered[index]))
|
||||
{
|
||||
lastFallbackIndex = index;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (lastFallbackIndex < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var lastFallback = ordered[lastFallbackIndex];
|
||||
var recovered = false;
|
||||
for (var index = lastFallbackIndex + 1; index < ordered.Length; index++)
|
||||
{
|
||||
if (IsRecoveryEvent(ordered[index]))
|
||||
{
|
||||
recovered = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (recovered)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
candidates.Add(new AlertCandidate(lastFallback.Query, 1, lastFallback.CreatedAt, lastFallback.CreatedAt));
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
private static bool IsFallbackAnswerEvent(SelfServeSignalEvent evt)
|
||||
{
|
||||
return evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase)
|
||||
&& (string.Equals(evt.AnswerStatus, "clarify", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(evt.AnswerStatus, "insufficient", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static bool IsRecoveryEvent(SelfServeSignalEvent evt)
|
||||
{
|
||||
return evt.EventType.Equals("click", StringComparison.OrdinalIgnoreCase)
|
||||
|| evt.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase)
|
||||
|| evt.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase)
|
||||
|| (evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(evt.AnswerStatus, "grounded", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static SelfServeSignalEvent[] OrderSessionEvents(IEnumerable<SelfServeSignalEvent> events)
|
||||
{
|
||||
return events
|
||||
.OrderBy(static evt => evt.CreatedAt)
|
||||
.ThenBy(static evt => evt.OrderKey, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? NullIfEmpty(string value)
|
||||
{
|
||||
return string.IsNullOrWhiteSpace(value) ? null : value;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<AlertCandidate>> LoadZeroResultCandidatesAsync(
|
||||
string tenantId,
|
||||
TimeSpan window,
|
||||
@@ -1170,12 +1434,26 @@ internal sealed class SearchQualityMetricsEntry
|
||||
public double ZeroResultRate { get; set; }
|
||||
public double AvgResultCount { get; set; }
|
||||
public double FeedbackScore { get; set; }
|
||||
public double FallbackAnswerRate { get; set; }
|
||||
public double ClarifyRate { get; set; }
|
||||
public double InsufficientRate { get; set; }
|
||||
public int ReformulationCount { get; set; }
|
||||
public int RescueActionCount { get; set; }
|
||||
public int AbandonedFallbackCount { get; set; }
|
||||
public string Period { get; set; } = "7d";
|
||||
public IReadOnlyList<SearchQualityLowQualityRow> LowQualityResults { get; set; } = [];
|
||||
public IReadOnlyList<SearchQualityTopQueryRow> TopQueries { get; set; } = [];
|
||||
public IReadOnlyList<SearchQualityTrendPoint> Trend { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed record SelfServeSignalEvent(
|
||||
string? SessionId,
|
||||
string EventType,
|
||||
string Query,
|
||||
string? AnswerStatus,
|
||||
DateTimeOffset CreatedAt,
|
||||
string OrderKey);
|
||||
|
||||
internal sealed class SearchQualityLowQualityRow
|
||||
{
|
||||
public string EntityKey { get; set; } = string.Empty;
|
||||
|
||||
@@ -107,18 +107,23 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
return EmptyResponse(string.Empty, request.K, "empty");
|
||||
}
|
||||
|
||||
if (query.Length > _unifiedOptions.MaxQueryLength)
|
||||
{
|
||||
return EmptyResponse(query, request.K, "query_too_long", request.Ambient);
|
||||
}
|
||||
|
||||
var tenantId = request.Filters?.Tenant ?? "global";
|
||||
var userId = request.Filters?.UserId ?? "anonymous";
|
||||
|
||||
if (query.Length > _unifiedOptions.MaxQueryLength)
|
||||
{
|
||||
var earlyResponse = EmptyResponse(query, request.K, "query_too_long", request.Ambient);
|
||||
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
|
||||
return earlyResponse;
|
||||
}
|
||||
|
||||
var tenantFlags = ResolveTenantFeatureFlags(tenantId);
|
||||
|
||||
if (!_options.Enabled || !IsSearchEnabledForTenant(tenantFlags) || string.IsNullOrWhiteSpace(_options.ConnectionString))
|
||||
{
|
||||
return EmptyResponse(query, request.K, "disabled", request.Ambient);
|
||||
var earlyResponse = EmptyResponse(query, request.K, "disabled", request.Ambient);
|
||||
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
|
||||
return earlyResponse;
|
||||
}
|
||||
|
||||
if (request.Ambient?.ResetSession == true &&
|
||||
@@ -321,6 +326,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
refinements,
|
||||
contextAnswer);
|
||||
|
||||
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan, response, cancellationToken).ConfigureAwait(false);
|
||||
EmitTelemetry(plan, response, tenantId);
|
||||
return response;
|
||||
}
|
||||
@@ -1858,4 +1864,40 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
||||
HasSuggestions: response.Suggestions is { Count: > 0 },
|
||||
HasRefinements: response.Refinements is { Count: > 0 }));
|
||||
}
|
||||
|
||||
private async Task RecordAnswerFrameAnalyticsAsync(
|
||||
string tenantId,
|
||||
string userId,
|
||||
UnifiedSearchRequest request,
|
||||
QueryPlan? plan,
|
||||
UnifiedSearchResponse response,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(response.Query))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var contextAnswer = response.ContextAnswer;
|
||||
if (contextAnswer is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var domain = ResolveContextDomain(plan, response.Cards, request.Ambient)
|
||||
?? response.Cards.FirstOrDefault()?.Domain
|
||||
?? request.Filters?.Domains?.FirstOrDefault();
|
||||
|
||||
await _analyticsService.RecordEventAsync(new SearchAnalyticsEvent(
|
||||
TenantId: tenantId,
|
||||
EventType: "answer_frame",
|
||||
Query: response.Query,
|
||||
UserId: userId,
|
||||
SessionId: request.Ambient?.SessionId,
|
||||
Domain: domain,
|
||||
ResultCount: response.Cards.Count,
|
||||
DurationMs: (int)Math.Min(int.MaxValue, Math.Max(0, response.Diagnostics.DurationMs)),
|
||||
AnswerStatus: contextAnswer.Status,
|
||||
AnswerCode: contextAnswer.Code), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1511,6 +1511,12 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
||||
"zero-result rate must be computed from raw zero_result analytics events");
|
||||
metrics.AvgResultCount.Should().BeApproximately(2.0, 0.2);
|
||||
metrics.FeedbackScore.Should().BeApproximately(50.0, 0.2);
|
||||
metrics.FallbackAnswerRate.Should().Be(0);
|
||||
metrics.ClarifyRate.Should().Be(0);
|
||||
metrics.InsufficientRate.Should().Be(0);
|
||||
metrics.ReformulationCount.Should().Be(0);
|
||||
metrics.RescueActionCount.Should().Be(0);
|
||||
metrics.AbandonedFallbackCount.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -1579,6 +1585,74 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
||||
.Should().BeTrue("low-quality rows should include repeated negative feedback entities");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task G10_SelfServeMetrics_IncludeFallbackReformulationAndRescueSignals()
|
||||
{
|
||||
var tenant = $"selfserve-metrics-{Guid.NewGuid():N}";
|
||||
using var writer = CreateAuthenticatedClient(tenant);
|
||||
using var admin = CreateAdminClient(tenant);
|
||||
|
||||
var analyticsResponse = await writer.PostAsJsonAsync("/v1/advisory-ai/search/analytics", new SearchAnalyticsApiRequest
|
||||
{
|
||||
Events =
|
||||
[
|
||||
new SearchAnalyticsApiEvent
|
||||
{
|
||||
EventType = "query",
|
||||
Query = "database connectivity",
|
||||
ResultCount = 0
|
||||
},
|
||||
new SearchAnalyticsApiEvent
|
||||
{
|
||||
EventType = "answer_frame",
|
||||
Query = "database connectivity",
|
||||
SessionId = "session-selfserve-1",
|
||||
Domain = "knowledge",
|
||||
AnswerStatus = "clarify",
|
||||
AnswerCode = "query_needs_scope"
|
||||
},
|
||||
new SearchAnalyticsApiEvent
|
||||
{
|
||||
EventType = "reformulation",
|
||||
Query = "postgresql connectivity",
|
||||
SessionId = "session-selfserve-1",
|
||||
Domain = "knowledge"
|
||||
},
|
||||
new SearchAnalyticsApiEvent
|
||||
{
|
||||
EventType = "answer_frame",
|
||||
Query = "postgresql connectivity",
|
||||
SessionId = "session-selfserve-1",
|
||||
Domain = "knowledge",
|
||||
AnswerStatus = "insufficient",
|
||||
AnswerCode = "no_grounded_evidence"
|
||||
},
|
||||
new SearchAnalyticsApiEvent
|
||||
{
|
||||
EventType = "rescue_action",
|
||||
Query = "postgresql connectivity",
|
||||
SessionId = "session-selfserve-1",
|
||||
Domain = "knowledge"
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
analyticsResponse.StatusCode.Should().Be(HttpStatusCode.NoContent);
|
||||
|
||||
var metricsResponse = await admin.GetAsync("/v1/advisory-ai/search/quality/metrics?period=7d");
|
||||
metricsResponse.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||
|
||||
var metrics = await metricsResponse.Content.ReadFromJsonAsync<SearchQualityMetricsDto>();
|
||||
metrics.Should().NotBeNull();
|
||||
metrics!.FallbackAnswerRate.Should().BeApproximately(100.0, 0.2);
|
||||
metrics.ClarifyRate.Should().BeApproximately(50.0, 0.2);
|
||||
metrics.InsufficientRate.Should().BeApproximately(50.0, 0.2);
|
||||
metrics.ReformulationCount.Should().Be(1);
|
||||
metrics.RescueActionCount.Should().Be(1);
|
||||
metrics.AbandonedFallbackCount.Should().Be(0,
|
||||
"a rescue action in the same session means the fallback flow was not abandoned");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task G10_AnalyticsEndpoint_ValidBatch_ReturnsNoContent()
|
||||
{
|
||||
@@ -1624,6 +1698,7 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
||||
EventType: "query",
|
||||
Query: query,
|
||||
UserId: userId,
|
||||
SessionId: "session-privacy-1",
|
||||
ResultCount: 1));
|
||||
|
||||
var events = analyticsService.GetFallbackEventsSnapshot(tenant, TimeSpan.FromMinutes(1));
|
||||
@@ -1634,6 +1709,7 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
||||
stored.Query.Should().NotContain("alice", "raw query text must not be stored in analytics events");
|
||||
stored.UserId.Should().Be(SearchAnalyticsPrivacy.HashUserId(tenant, userId));
|
||||
stored.UserId.Should().NotBe(userId, "user identifiers must be pseudonymized before persistence");
|
||||
stored.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId(tenant, "session-privacy-1"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -1862,6 +1938,76 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
||||
.Should().BeTrue("five repeated zero-result events should create a zero_result quality alert");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task G10_FallbackLoopSessions_CreateQualityAlert()
|
||||
{
|
||||
var tenant = $"fallback-loop-{Guid.NewGuid():N}";
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
|
||||
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
|
||||
|
||||
await analytics.RecordEventsAsync(
|
||||
[
|
||||
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate", SessionId: "sess-loop-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
|
||||
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate release", SessionId: "sess-loop-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
|
||||
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate release production", SessionId: "sess-loop-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence")
|
||||
]);
|
||||
|
||||
await monitor.RefreshAlertsAsync(tenant);
|
||||
var alerts = await monitor.GetAlertsAsync(tenant, status: "open", alertType: "fallback_loop");
|
||||
|
||||
var expectedQueryHash = SearchAnalyticsPrivacy.HashQuery("policy gate release production");
|
||||
alerts.Any(alert => alert.Query.Equals(expectedQueryHash, StringComparison.OrdinalIgnoreCase))
|
||||
.Should().BeTrue("three fallback answer frames in the same session should create a fallback_loop alert");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task G10_AbandonedFallbackSessions_CreateQualityAlert()
|
||||
{
|
||||
var tenant = $"abandoned-fallback-{Guid.NewGuid():N}";
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
|
||||
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
|
||||
|
||||
await analytics.RecordEventsAsync(
|
||||
[
|
||||
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
|
||||
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-2", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
|
||||
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-3", AnswerStatus: "clarify", AnswerCode: "query_needs_scope")
|
||||
]);
|
||||
|
||||
await monitor.RefreshAlertsAsync(tenant);
|
||||
var alerts = await monitor.GetAlertsAsync(tenant, status: "open", alertType: "abandoned_fallback");
|
||||
|
||||
var expectedQueryHash = SearchAnalyticsPrivacy.HashQuery("unknown control token");
|
||||
alerts.Any(alert => alert.Query.Equals(expectedQueryHash, StringComparison.OrdinalIgnoreCase))
|
||||
.Should().BeTrue("repeated sessions that end in fallback without recovery should create an abandoned_fallback alert");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task G10_RecoveredFallbackSessions_DoNotCountAsAbandoned()
|
||||
{
|
||||
var tenant = $"recovered-fallback-{Guid.NewGuid():N}";
|
||||
using var scope = _factory.Services.CreateScope();
|
||||
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
|
||||
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
|
||||
|
||||
await analytics.RecordEventsAsync(
|
||||
[
|
||||
new SearchAnalyticsEvent(tenant, "answer_frame", "database connectivity", SessionId: "sess-recovered-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
|
||||
new SearchAnalyticsEvent(tenant, "reformulation", "postgresql connectivity", SessionId: "sess-recovered-1"),
|
||||
new SearchAnalyticsEvent(tenant, "answer_frame", "postgresql connectivity", SessionId: "sess-recovered-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
|
||||
new SearchAnalyticsEvent(tenant, "rescue_action", "postgresql connectivity", SessionId: "sess-recovered-1")
|
||||
]);
|
||||
|
||||
var metrics = await monitor.GetMetricsAsync(tenant, "7d");
|
||||
|
||||
metrics.ReformulationCount.Should().Be(1);
|
||||
metrics.RescueActionCount.Should().Be(1);
|
||||
metrics.AbandonedFallbackCount.Should().Be(0,
|
||||
"a session with a reformulation and rescue action after fallback is a recovery path, not an abandonment");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task G10_NegativeFeedbackBurst_CreatesHighNegativeFeedbackAlert()
|
||||
{
|
||||
|
||||
@@ -22,4 +22,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
|
||||
| SPRINT_20260224_G1-004-BENCH | DONE | Semantic recall benchmark: 13 tests in `SemanticRecallBenchmarkTests.cs`, 48-query fixture (`semantic-recall-benchmark.json`), `SemanticRecallBenchmarkStore` (33 chunks), `SemanticSimulationEncoder` (40+ semantic groups). Semantic strictly outperforms hash on synonym queries. |
|
||||
|
||||
| AI-SELF-005 | DONE | Integration coverage now asserts grounded, clarify, and insufficient contextual-answer states through the real endpoint contract. |
|
||||
| AI-SELF-004 | DONE | Targeted telemetry coverage now verifies fallback metrics, recovered-session suppression of abandoned counts, session hashing, and self-serve alert generation with xUnit v3-compatible runner commands. |
|
||||
| AI-SELF-006 | DONE | Verification includes a real local corpus rebuild and a live query assertion, not only test doubles. |
|
||||
|
||||
@@ -8,6 +8,7 @@ using StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
|
||||
using StellaOps.AdvisoryAI.UnifiedSearch.QueryUnderstanding;
|
||||
using StellaOps.AdvisoryAI.UnifiedSearch.Synthesis;
|
||||
using StellaOps.AdvisoryAI.Vectorization;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using Xunit;
|
||||
|
||||
@@ -124,6 +125,101 @@ public sealed class UnifiedSearchServiceTests
|
||||
result.ContextAnswer.Questions!.Should().OnlyContain(question => question.Kind == "recover");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SearchAsync_records_answer_frame_analytics_for_clarify_state()
|
||||
{
|
||||
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
|
||||
{
|
||||
Enabled = true,
|
||||
ConnectionString = "Host=localhost;Database=test"
|
||||
});
|
||||
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
|
||||
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
|
||||
var service = CreateService(
|
||||
storeMock: CreateEmptyStoreMock(),
|
||||
analyticsService: analyticsService,
|
||||
qualityMonitor: qualityMonitor);
|
||||
|
||||
var result = await service.SearchAsync(
|
||||
new UnifiedSearchRequest(
|
||||
"status",
|
||||
Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" },
|
||||
Ambient: new AmbientContext
|
||||
{
|
||||
CurrentRoute = "/ops/operations/doctor",
|
||||
SessionId = "session-123"
|
||||
}),
|
||||
CancellationToken.None);
|
||||
|
||||
result.ContextAnswer.Should().NotBeNull();
|
||||
result.ContextAnswer!.Status.Should().Be("clarify");
|
||||
|
||||
var events = analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1));
|
||||
events.Should().ContainSingle();
|
||||
var answerFrame = events.Single().Event;
|
||||
answerFrame.EventType.Should().Be("answer_frame");
|
||||
answerFrame.AnswerStatus.Should().Be("clarify");
|
||||
answerFrame.AnswerCode.Should().Be(result.ContextAnswer.Code);
|
||||
answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-123"));
|
||||
answerFrame.Query.Should().Be(SearchAnalyticsPrivacy.HashQuery("status"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SearchAsync_records_answer_frame_analytics_for_grounded_state()
|
||||
{
|
||||
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
|
||||
{
|
||||
Enabled = true,
|
||||
ConnectionString = "Host=localhost;Database=test"
|
||||
});
|
||||
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
|
||||
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
|
||||
|
||||
var doctorRow = MakeRow(
|
||||
"chunk-doctor-analytics",
|
||||
"doctor_check",
|
||||
"PostgreSQL connectivity",
|
||||
JsonDocument.Parse("{\"domain\":\"knowledge\",\"checkCode\":\"check.core.db.connectivity\"}"));
|
||||
|
||||
var storeMock = new Mock<IKnowledgeSearchStore>();
|
||||
storeMock.Setup(s => s.SearchFtsAsync(
|
||||
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>(), It.IsAny<string?>()))
|
||||
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow });
|
||||
storeMock.Setup(s => s.LoadVectorCandidatesAsync(
|
||||
It.IsAny<float[]>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync([]);
|
||||
|
||||
var service = CreateService(
|
||||
storeMock: storeMock,
|
||||
analyticsService: analyticsService,
|
||||
qualityMonitor: qualityMonitor);
|
||||
|
||||
var result = await service.SearchAsync(
|
||||
new UnifiedSearchRequest(
|
||||
"database connectivity",
|
||||
Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" },
|
||||
Ambient: new AmbientContext
|
||||
{
|
||||
CurrentRoute = "/ops/operations/doctor",
|
||||
SessionId = "session-456"
|
||||
}),
|
||||
CancellationToken.None);
|
||||
|
||||
result.ContextAnswer.Should().NotBeNull();
|
||||
result.ContextAnswer!.Status.Should().Be("grounded");
|
||||
|
||||
var events = analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1));
|
||||
events.Should().ContainSingle();
|
||||
var answerFrame = events.Single().Event;
|
||||
answerFrame.EventType.Should().Be("answer_frame");
|
||||
answerFrame.AnswerStatus.Should().Be("grounded");
|
||||
answerFrame.AnswerCode.Should().Be("retrieved_evidence");
|
||||
answerFrame.ResultCount.Should().Be(1);
|
||||
answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-456"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SearchAsync_returns_empty_when_tenant_feature_flag_disables_search()
|
||||
{
|
||||
@@ -827,7 +923,9 @@ public sealed class UnifiedSearchServiceTests
|
||||
private static UnifiedSearchService CreateService(
|
||||
bool enabled = true,
|
||||
Mock<IKnowledgeSearchStore>? storeMock = null,
|
||||
UnifiedSearchOptions? unifiedOptions = null)
|
||||
UnifiedSearchOptions? unifiedOptions = null,
|
||||
SearchAnalyticsService? analyticsService = null,
|
||||
SearchQualityMonitor? qualityMonitor = null)
|
||||
{
|
||||
var options = Options.Create(new KnowledgeSearchOptions
|
||||
{
|
||||
@@ -855,8 +953,8 @@ public sealed class UnifiedSearchServiceTests
|
||||
var weightCalculator = new DomainWeightCalculator(extractor, classifier, options);
|
||||
var planBuilder = new QueryPlanBuilder(extractor, classifier, weightCalculator);
|
||||
var synthesisEngine = new SynthesisTemplateEngine();
|
||||
var analyticsService = new SearchAnalyticsService(options, NullLogger<SearchAnalyticsService>.Instance);
|
||||
var qualityMonitor = new SearchQualityMonitor(options, NullLogger<SearchQualityMonitor>.Instance);
|
||||
analyticsService ??= new SearchAnalyticsService(options, NullLogger<SearchAnalyticsService>.Instance);
|
||||
qualityMonitor ??= new SearchQualityMonitor(options, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
|
||||
var entityAliasService = new Mock<IEntityAliasService>();
|
||||
entityAliasService.Setup(s => s.ResolveAliasesAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(Array.Empty<(string EntityKey, string EntityType)>());
|
||||
|
||||
Reference in New Issue
Block a user