Add self-serve search telemetry gap surfacing
This commit is contained in:
@@ -65,7 +65,7 @@ Completion criteria:
|
|||||||
- [ ] Ambiguous queries return clarifying prompts instead of a blank answer slot.
|
- [ ] Ambiguous queries return clarifying prompts instead of a blank answer slot.
|
||||||
|
|
||||||
### AI-SELF-004 - Self-serve telemetry and gap surfacing
|
### AI-SELF-004 - Self-serve telemetry and gap surfacing
|
||||||
Status: TODO
|
Status: DONE
|
||||||
Dependency: AI-SELF-002
|
Dependency: AI-SELF-002
|
||||||
Owners: Developer (AdvisoryAI), Test Automation
|
Owners: Developer (AdvisoryAI), Test Automation
|
||||||
Task description:
|
Task description:
|
||||||
@@ -73,9 +73,9 @@ Task description:
|
|||||||
- Expose enough structured data to drive a gap-closure backlog.
|
- Expose enough structured data to drive a gap-closure backlog.
|
||||||
|
|
||||||
Completion criteria:
|
Completion criteria:
|
||||||
- [ ] Telemetry captures unanswered and reformulated journeys without persisting raw sensitive prompts unnecessarily.
|
- [x] Telemetry captures unanswered and reformulated journeys without persisting raw sensitive prompts unnecessarily.
|
||||||
- [ ] Operational docs explain how to review self-serve gaps.
|
- [x] Operational docs explain how to review self-serve gaps.
|
||||||
- [ ] Tests cover telemetry emission for fallback paths.
|
- [x] Tests cover telemetry emission for fallback paths.
|
||||||
|
|
||||||
### AI-SELF-005 - Targeted behavioral verification
|
### AI-SELF-005 - Targeted behavioral verification
|
||||||
Status: DONE
|
Status: DONE
|
||||||
@@ -110,6 +110,8 @@ Completion criteria:
|
|||||||
| 2026-03-07 | Implemented `contextAnswer` in unified search/backend API mapping, added deterministic `grounded` / `clarify` / `insufficient` rules plus follow-up question generation, and extended telemetry fields for answer-state visibility. | Developer |
|
| 2026-03-07 | Implemented `contextAnswer` in unified search/backend API mapping, added deterministic `grounded` / `clarify` / `insufficient` rules plus follow-up question generation, and extended telemetry fields for answer-state visibility. | Developer |
|
||||||
| 2026-03-07 | Verified the AdvisoryAI test project after the contract change with `dotnet test "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj" --no-restore -v normal` (`877/877` passing). | Test Automation |
|
| 2026-03-07 | Verified the AdvisoryAI test project after the contract change with `dotnet test "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj" --no-restore -v normal` (`877/877` passing). | Test Automation |
|
||||||
| 2026-03-07 | Exercised the live rebuilt-corpus lane against `http://127.0.0.1:10451`: `POST /v1/advisory-ai/index/rebuild`, `POST /v1/search/index/rebuild`, then `POST /v1/search/query` for `database connectivity`, which returned `contextAnswer.status = grounded`, 3 citations, and 10 cards over ingested data. | Test Automation |
|
| 2026-03-07 | Exercised the live rebuilt-corpus lane against `http://127.0.0.1:10451`: `POST /v1/advisory-ai/index/rebuild`, `POST /v1/search/index/rebuild`, then `POST /v1/search/query` for `database connectivity`, which returned `contextAnswer.status = grounded`, 3 citations, and 10 cards over ingested data. | Test Automation |
|
||||||
|
| 2026-03-07 | Implemented optional self-serve analytics columns (`session_id`, `answer_status`, `answer_code`), metrics (`fallbackAnswerRate`, `clarifyRate`, `insufficientRate`, `reformulationCount`, `rescueActionCount`, `abandonedFallbackCount`), and alerting (`fallback_loop`, `abandoned_fallback`) with privacy-preserving session hashing. | Developer |
|
||||||
|
| 2026-03-07 | Verified targeted telemetry coverage with xUnit v3 runner commands against `UnifiedSearchSprintIntegrationTests` and added a recovered-session regression test so rescue actions clear abandonment as expected. | Test Automation |
|
||||||
|
|
||||||
## Decisions & Risks
|
## Decisions & Risks
|
||||||
- Decision: the backend contract must return explicit answer states instead of leaving the UI to infer confidence from cards alone.
|
- Decision: the backend contract must return explicit answer states instead of leaving the UI to infer confidence from cards alone.
|
||||||
@@ -122,6 +124,8 @@ Completion criteria:
|
|||||||
- Risk: mocked endpoint tests can overstate confidence if ingestion adapters or corpus rebuild order drift.
|
- Risk: mocked endpoint tests can overstate confidence if ingestion adapters or corpus rebuild order drift.
|
||||||
- Mitigation: keep rebuild order documented, execute it during verification, and record which routes have live-ingested parity.
|
- Mitigation: keep rebuild order documented, execute it during verification, and record which routes have live-ingested parity.
|
||||||
- Decision: `stella advisoryai sources prepare` is optional for local verification when checked-in Doctor seed/control files are already sufficient, but it requires `STELLAOPS_BACKEND_URL` whenever live Doctor discovery is expected.
|
- Decision: `stella advisoryai sources prepare` is optional for local verification when checked-in Doctor seed/control files are already sufficient, but it requires `STELLAOPS_BACKEND_URL` whenever live Doctor discovery is expected.
|
||||||
|
- Decision: self-serve analytics is optional and additive; search behavior must not depend on telemetry event emission.
|
||||||
|
- Decision: targeted AdvisoryAI verification uses xUnit v3 / Microsoft.Testing.Platform-compatible filters (`-- --filter-class` or the built test executable), not legacy VSTest `--filter`.
|
||||||
|
|
||||||
## Next Checkpoints
|
## Next Checkpoints
|
||||||
- 2026-03-10: Freeze answer payload shape and fallback taxonomy.
|
- 2026-03-10: Freeze answer payload shape and fallback taxonomy.
|
||||||
|
|||||||
@@ -17,7 +17,10 @@ public static class SearchAnalyticsEndpoints
|
|||||||
"query",
|
"query",
|
||||||
"click",
|
"click",
|
||||||
"zero_result",
|
"zero_result",
|
||||||
"synthesis"
|
"synthesis",
|
||||||
|
"answer_frame",
|
||||||
|
"reformulation",
|
||||||
|
"rescue_action"
|
||||||
};
|
};
|
||||||
|
|
||||||
public static RouteGroupBuilder MapSearchAnalyticsEndpoints(this IEndpointRouteBuilder builder)
|
public static RouteGroupBuilder MapSearchAnalyticsEndpoints(this IEndpointRouteBuilder builder)
|
||||||
@@ -30,10 +33,11 @@ public static class SearchAnalyticsEndpoints
|
|||||||
|
|
||||||
group.MapPost("/analytics", RecordAnalyticsAsync)
|
group.MapPost("/analytics", RecordAnalyticsAsync)
|
||||||
.WithName("SearchAnalyticsRecord")
|
.WithName("SearchAnalyticsRecord")
|
||||||
.WithSummary("Records batch search analytics events (query, click, zero_result, synthesis).")
|
.WithSummary("Records batch search analytics events (query, click, zero_result, synthesis, answer_frame, reformulation, rescue_action).")
|
||||||
.WithDescription(
|
.WithDescription(
|
||||||
"Accepts a batch of search analytics events for tracking query frequency, click-through rates, " +
|
"Accepts a batch of search analytics events for tracking query frequency, click-through rates, " +
|
||||||
"zero-result queries, and synthesis usage. Queries and user identifiers are pseudonymized before persistence. " +
|
"zero-result queries, self-serve answer states, reformulations, rescue-action usage, and synthesis usage. " +
|
||||||
|
"Queries and user/session identifiers are pseudonymized before persistence. " +
|
||||||
"Fire-and-forget from the client; failures do not affect search functionality.")
|
"Fire-and-forget from the client; failures do not affect search functionality.")
|
||||||
.RequireAuthorization(AdvisoryAIPolicies.OperatePolicy)
|
.RequireAuthorization(AdvisoryAIPolicies.OperatePolicy)
|
||||||
.Produces(StatusCodes.Status204NoContent)
|
.Produces(StatusCodes.Status204NoContent)
|
||||||
@@ -113,11 +117,14 @@ public static class SearchAnalyticsEndpoints
|
|||||||
EventType: apiEvent.EventType.Trim().ToLowerInvariant(),
|
EventType: apiEvent.EventType.Trim().ToLowerInvariant(),
|
||||||
Query: apiEvent.Query.Trim(),
|
Query: apiEvent.Query.Trim(),
|
||||||
UserId: userId,
|
UserId: userId,
|
||||||
|
SessionId: string.IsNullOrWhiteSpace(apiEvent.SessionId) ? null : apiEvent.SessionId.Trim(),
|
||||||
EntityKey: string.IsNullOrWhiteSpace(apiEvent.EntityKey) ? null : apiEvent.EntityKey.Trim(),
|
EntityKey: string.IsNullOrWhiteSpace(apiEvent.EntityKey) ? null : apiEvent.EntityKey.Trim(),
|
||||||
Domain: string.IsNullOrWhiteSpace(apiEvent.Domain) ? null : apiEvent.Domain.Trim(),
|
Domain: string.IsNullOrWhiteSpace(apiEvent.Domain) ? null : apiEvent.Domain.Trim(),
|
||||||
ResultCount: apiEvent.ResultCount,
|
ResultCount: apiEvent.ResultCount,
|
||||||
Position: apiEvent.Position,
|
Position: apiEvent.Position,
|
||||||
DurationMs: apiEvent.DurationMs));
|
DurationMs: apiEvent.DurationMs,
|
||||||
|
AnswerStatus: string.IsNullOrWhiteSpace(apiEvent.AnswerStatus) ? null : apiEvent.AnswerStatus.Trim(),
|
||||||
|
AnswerCode: string.IsNullOrWhiteSpace(apiEvent.AnswerCode) ? null : apiEvent.AnswerCode.Trim()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (events.Count > 0)
|
if (events.Count > 0)
|
||||||
@@ -301,6 +308,8 @@ public sealed record SearchAnalyticsApiEvent
|
|||||||
|
|
||||||
public string Query { get; init; } = string.Empty;
|
public string Query { get; init; } = string.Empty;
|
||||||
|
|
||||||
|
public string? SessionId { get; init; }
|
||||||
|
|
||||||
public string? EntityKey { get; init; }
|
public string? EntityKey { get; init; }
|
||||||
|
|
||||||
public string? Domain { get; init; }
|
public string? Domain { get; init; }
|
||||||
@@ -310,6 +319,10 @@ public sealed record SearchAnalyticsApiEvent
|
|||||||
public int? Position { get; init; }
|
public int? Position { get; init; }
|
||||||
|
|
||||||
public int? DurationMs { get; init; }
|
public int? DurationMs { get; init; }
|
||||||
|
|
||||||
|
public string? AnswerStatus { get; init; }
|
||||||
|
|
||||||
|
public string? AnswerCode { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed record SearchHistoryApiResponse
|
public sealed record SearchHistoryApiResponse
|
||||||
|
|||||||
@@ -39,11 +39,11 @@ public static class SearchFeedbackEndpoints
|
|||||||
// G10-002: List quality alerts (admin only)
|
// G10-002: List quality alerts (admin only)
|
||||||
group.MapGet("/quality/alerts", GetAlertsAsync)
|
group.MapGet("/quality/alerts", GetAlertsAsync)
|
||||||
.WithName("SearchQualityAlertsList")
|
.WithName("SearchQualityAlertsList")
|
||||||
.WithSummary("Lists open search quality alerts (zero-result queries, high negative feedback).")
|
.WithSummary("Lists open search quality alerts (zero-result, high negative feedback, fallback loops, abandoned fallback).")
|
||||||
.WithDescription(
|
.WithDescription(
|
||||||
"Returns search quality alerts ordered by occurrence count. " +
|
"Returns search quality alerts ordered by occurrence count. " +
|
||||||
"Filterable by status (open, acknowledged, resolved) and alert type " +
|
"Filterable by status (open, acknowledged, resolved) and alert type " +
|
||||||
"(zero_result, low_feedback, high_negative_feedback). Requires admin scope.")
|
"(zero_result, high_negative_feedback, fallback_loop, abandoned_fallback). Requires admin scope.")
|
||||||
.RequireAuthorization(AdvisoryAIPolicies.AdminPolicy)
|
.RequireAuthorization(AdvisoryAIPolicies.AdminPolicy)
|
||||||
.Produces<IReadOnlyList<SearchQualityAlertDto>>(StatusCodes.Status200OK)
|
.Produces<IReadOnlyList<SearchQualityAlertDto>>(StatusCodes.Status200OK)
|
||||||
.Produces(StatusCodes.Status403Forbidden);
|
.Produces(StatusCodes.Status403Forbidden);
|
||||||
@@ -198,6 +198,12 @@ public static class SearchFeedbackEndpoints
|
|||||||
ZeroResultRate = metrics.ZeroResultRate,
|
ZeroResultRate = metrics.ZeroResultRate,
|
||||||
AvgResultCount = metrics.AvgResultCount,
|
AvgResultCount = metrics.AvgResultCount,
|
||||||
FeedbackScore = metrics.FeedbackScore,
|
FeedbackScore = metrics.FeedbackScore,
|
||||||
|
FallbackAnswerRate = metrics.FallbackAnswerRate,
|
||||||
|
ClarifyRate = metrics.ClarifyRate,
|
||||||
|
InsufficientRate = metrics.InsufficientRate,
|
||||||
|
ReformulationCount = metrics.ReformulationCount,
|
||||||
|
RescueActionCount = metrics.RescueActionCount,
|
||||||
|
AbandonedFallbackCount = metrics.AbandonedFallbackCount,
|
||||||
Period = metrics.Period,
|
Period = metrics.Period,
|
||||||
LowQualityResults = metrics.LowQualityResults
|
LowQualityResults = metrics.LowQualityResults
|
||||||
.Select(row => new SearchLowQualityResultDto
|
.Select(row => new SearchLowQualityResultDto
|
||||||
@@ -308,6 +314,12 @@ public sealed record SearchQualityMetricsDto
|
|||||||
public double ZeroResultRate { get; init; }
|
public double ZeroResultRate { get; init; }
|
||||||
public double AvgResultCount { get; init; }
|
public double AvgResultCount { get; init; }
|
||||||
public double FeedbackScore { get; init; }
|
public double FeedbackScore { get; init; }
|
||||||
|
public double FallbackAnswerRate { get; init; }
|
||||||
|
public double ClarifyRate { get; init; }
|
||||||
|
public double InsufficientRate { get; init; }
|
||||||
|
public int ReformulationCount { get; init; }
|
||||||
|
public int RescueActionCount { get; init; }
|
||||||
|
public int AbandonedFallbackCount { get; init; }
|
||||||
public string Period { get; init; } = "7d";
|
public string Period { get; init; } = "7d";
|
||||||
public IReadOnlyList<SearchLowQualityResultDto> LowQualityResults { get; init; } = [];
|
public IReadOnlyList<SearchLowQualityResultDto> LowQualityResults { get; init; } = [];
|
||||||
public IReadOnlyList<SearchTopQueryDto> TopQueries { get; init; } = [];
|
public IReadOnlyList<SearchTopQueryDto> TopQueries { get; init; } = [];
|
||||||
|
|||||||
@@ -17,4 +17,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
|
|||||||
| SPRINT_20260224_G1-G10 | DONE | Search improvement sprints G1–G10 implemented. New endpoints: `SearchAnalyticsEndpoints.cs` (history, events, popularity), `SearchFeedbackEndpoints.cs` (feedback, quality alerts, metrics). Extended: `UnifiedSearchEndpoints.cs` (suggestions, refinements, previews, diagnostics.activeEncoder). Extended: `KnowledgeSearchEndpoints.cs` (activeEncoder in diagnostics). See `docs/modules/advisory-ai/knowledge-search.md` for full testing guide. |
|
| SPRINT_20260224_G1-G10 | DONE | Search improvement sprints G1–G10 implemented. New endpoints: `SearchAnalyticsEndpoints.cs` (history, events, popularity), `SearchFeedbackEndpoints.cs` (feedback, quality alerts, metrics). Extended: `UnifiedSearchEndpoints.cs` (suggestions, refinements, previews, diagnostics.activeEncoder). Extended: `KnowledgeSearchEndpoints.cs` (activeEncoder in diagnostics). See `docs/modules/advisory-ai/knowledge-search.md` for full testing guide. |
|
||||||
|
|
||||||
| AI-SELF-001 | DONE | Unified search endpoint contract now exposes backend contextual answer fields for self-serve search. |
|
| AI-SELF-001 | DONE | Unified search endpoint contract now exposes backend contextual answer fields for self-serve search. |
|
||||||
|
| AI-SELF-004 | DONE | Search analytics and quality endpoints now surface self-serve metrics and alerts (`fallback_loop`, `abandoned_fallback`) while keeping telemetry optional. |
|
||||||
| AI-SELF-006 | DONE | Endpoint readiness now includes a proven local rebuilt-corpus verification lane in addition to stubbed integration tests. |
|
| AI-SELF-006 | DONE | Endpoint readiness now includes a proven local rebuilt-corpus verification lane in addition to stubbed integration tests. |
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
-- 008_search_self_serve_analytics.sql
|
||||||
|
-- Adds self-serve telemetry fields to search_events so fallback loops,
|
||||||
|
-- reformulations, rescue actions, and abandoned fallback sessions can be tracked.
|
||||||
|
|
||||||
|
ALTER TABLE advisoryai.search_events
|
||||||
|
ADD COLUMN IF NOT EXISTS session_id TEXT;
|
||||||
|
|
||||||
|
ALTER TABLE advisoryai.search_events
|
||||||
|
ADD COLUMN IF NOT EXISTS answer_status TEXT;
|
||||||
|
|
||||||
|
ALTER TABLE advisoryai.search_events
|
||||||
|
ADD COLUMN IF NOT EXISTS answer_code TEXT;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_search_events_tenant_session
|
||||||
|
ON advisoryai.search_events (tenant_id, session_id, created_at)
|
||||||
|
WHERE session_id IS NOT NULL;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_search_events_answer_status
|
||||||
|
ON advisoryai.search_events (tenant_id, answer_status, created_at)
|
||||||
|
WHERE answer_status IS NOT NULL;
|
||||||
@@ -13,7 +13,7 @@ Source of truth: `docs/implplan/SPRINT_20260113_005_ADVISORYAI_controlled_conver
|
|||||||
| AI-SELF-001 | DONE | Unified search now emits the contextual answer payload (`contextAnswer`) with answer state, citations, and follow-up questions for self-serve search. |
|
| AI-SELF-001 | DONE | Unified search now emits the contextual answer payload (`contextAnswer`) with answer state, citations, and follow-up questions for self-serve search. |
|
||||||
| AI-SELF-002 | DONE | Deterministic grounded/clarify/insufficient fallback policy is implemented in unified search orchestration. |
|
| AI-SELF-002 | DONE | Deterministic grounded/clarify/insufficient fallback policy is implemented in unified search orchestration. |
|
||||||
| AI-SELF-003 | DONE | Follow-up question generation from route/domain intent, recent actions, and evidence is implemented. |
|
| AI-SELF-003 | DONE | Follow-up question generation from route/domain intent, recent actions, and evidence is implemented. |
|
||||||
| AI-SELF-004 | TODO | Telemetry for unanswered and reformulated self-serve journeys is still pending. |
|
| AI-SELF-004 | DONE | Optional self-serve telemetry now captures answer frames, reformulations, rescue actions, and abandonment signals via hashed session ids; quality metrics and alerts expose the gaps operationally. |
|
||||||
| AI-SELF-006 | DONE | Live ingestion-backed answer verification succeeded on the Doctor/knowledge route after local rebuild. |
|
| AI-SELF-006 | DONE | Live ingestion-backed answer verification succeeded on the Doctor/knowledge route after local rebuild. |
|
||||||
| SPRINT_20260222_051-AKS-INGEST | DONE | Added deterministic AKS ingestion controls: markdown allow-list manifest loading, OpenAPI aggregate source path support, and doctor control projection integration for search chunks, including fallback doctor metadata hydration from controls projection fields. |
|
| SPRINT_20260222_051-AKS-INGEST | DONE | Added deterministic AKS ingestion controls: markdown allow-list manifest loading, OpenAPI aggregate source path support, and doctor control projection integration for search chunks, including fallback doctor metadata hydration from controls projection fields. |
|
||||||
| AUDIT-0017-M | DONE | Maintainability audit for StellaOps.AdvisoryAI. |
|
| AUDIT-0017-M | DONE | Maintainability audit for StellaOps.AdvisoryAI. |
|
||||||
|
|||||||
@@ -32,6 +32,19 @@ internal static class SearchAnalyticsPrivacy
|
|||||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static string? HashSessionId(string tenantId, string? sessionId)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(sessionId))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var normalizedTenant = tenantId.Trim().ToLowerInvariant();
|
||||||
|
var normalizedSession = sessionId.Trim().ToLowerInvariant();
|
||||||
|
var hash = SHA256.HashData(Encoding.UTF8.GetBytes($"{normalizedTenant}|session|{normalizedSession}"));
|
||||||
|
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||||
|
}
|
||||||
|
|
||||||
public static string? RedactFreeform(string? value)
|
public static string? RedactFreeform(string? value)
|
||||||
{
|
{
|
||||||
_ = value;
|
_ = value;
|
||||||
|
|||||||
@@ -37,11 +37,14 @@ internal sealed class SearchAnalyticsService
|
|||||||
await conn.OpenAsync(ct).ConfigureAwait(false);
|
await conn.OpenAsync(ct).ConfigureAwait(false);
|
||||||
|
|
||||||
await using var cmd = new NpgsqlCommand(@"
|
await using var cmd = new NpgsqlCommand(@"
|
||||||
INSERT INTO advisoryai.search_events (tenant_id, user_id, event_type, query, entity_key, domain, result_count, position, duration_ms)
|
INSERT INTO advisoryai.search_events
|
||||||
VALUES (@tenant_id, @user_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms)", conn);
|
(tenant_id, user_id, session_id, event_type, query, entity_key, domain, result_count, position, duration_ms, answer_status, answer_code)
|
||||||
|
VALUES
|
||||||
|
(@tenant_id, @user_id, @session_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms, @answer_status, @answer_code)", conn);
|
||||||
|
|
||||||
cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId);
|
cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId);
|
||||||
cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value);
|
||||||
|
cmd.Parameters.AddWithValue("session_id", (object?)persistedEvent.SessionId ?? DBNull.Value);
|
||||||
cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType);
|
cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType);
|
||||||
cmd.Parameters.AddWithValue("query", persistedEvent.Query);
|
cmd.Parameters.AddWithValue("query", persistedEvent.Query);
|
||||||
cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value);
|
||||||
@@ -49,6 +52,8 @@ internal sealed class SearchAnalyticsService
|
|||||||
cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value);
|
||||||
cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value);
|
||||||
cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value);
|
||||||
|
cmd.Parameters.AddWithValue("answer_status", (object?)persistedEvent.AnswerStatus ?? DBNull.Value);
|
||||||
|
cmd.Parameters.AddWithValue("answer_code", (object?)persistedEvent.AnswerCode ?? DBNull.Value);
|
||||||
|
|
||||||
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||||
RecordFallbackEvent(persistedEvent, recordedAt);
|
RecordFallbackEvent(persistedEvent, recordedAt);
|
||||||
@@ -88,11 +93,14 @@ internal sealed class SearchAnalyticsService
|
|||||||
var persistedEvent = SanitizeEvent(evt);
|
var persistedEvent = SanitizeEvent(evt);
|
||||||
|
|
||||||
await using var cmd = new NpgsqlCommand(@"
|
await using var cmd = new NpgsqlCommand(@"
|
||||||
INSERT INTO advisoryai.search_events (tenant_id, user_id, event_type, query, entity_key, domain, result_count, position, duration_ms)
|
INSERT INTO advisoryai.search_events
|
||||||
VALUES (@tenant_id, @user_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms)", conn);
|
(tenant_id, user_id, session_id, event_type, query, entity_key, domain, result_count, position, duration_ms, answer_status, answer_code)
|
||||||
|
VALUES
|
||||||
|
(@tenant_id, @user_id, @session_id, @event_type, @query, @entity_key, @domain, @result_count, @position, @duration_ms, @answer_status, @answer_code)", conn);
|
||||||
|
|
||||||
cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId);
|
cmd.Parameters.AddWithValue("tenant_id", persistedEvent.TenantId);
|
||||||
cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("user_id", (object?)persistedEvent.UserId ?? DBNull.Value);
|
||||||
|
cmd.Parameters.AddWithValue("session_id", (object?)persistedEvent.SessionId ?? DBNull.Value);
|
||||||
cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType);
|
cmd.Parameters.AddWithValue("event_type", persistedEvent.EventType);
|
||||||
cmd.Parameters.AddWithValue("query", persistedEvent.Query);
|
cmd.Parameters.AddWithValue("query", persistedEvent.Query);
|
||||||
cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("entity_key", (object?)persistedEvent.EntityKey ?? DBNull.Value);
|
||||||
@@ -100,6 +108,8 @@ internal sealed class SearchAnalyticsService
|
|||||||
cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("result_count", (object?)persistedEvent.ResultCount ?? DBNull.Value);
|
||||||
cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("position", (object?)persistedEvent.Position ?? DBNull.Value);
|
||||||
cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value);
|
cmd.Parameters.AddWithValue("duration_ms", (object?)persistedEvent.DurationMs ?? DBNull.Value);
|
||||||
|
cmd.Parameters.AddWithValue("answer_status", (object?)persistedEvent.AnswerStatus ?? DBNull.Value);
|
||||||
|
cmd.Parameters.AddWithValue("answer_code", (object?)persistedEvent.AnswerCode ?? DBNull.Value);
|
||||||
|
|
||||||
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||||
RecordFallbackEvent(persistedEvent, recordedAt);
|
RecordFallbackEvent(persistedEvent, recordedAt);
|
||||||
@@ -637,8 +647,16 @@ internal sealed class SearchAnalyticsService
|
|||||||
{
|
{
|
||||||
return evt with
|
return evt with
|
||||||
{
|
{
|
||||||
|
EventType = evt.EventType.Trim().ToLowerInvariant(),
|
||||||
Query = SearchAnalyticsPrivacy.HashQuery(evt.Query),
|
Query = SearchAnalyticsPrivacy.HashQuery(evt.Query),
|
||||||
UserId = SearchAnalyticsPrivacy.HashUserId(evt.TenantId, evt.UserId)
|
UserId = SearchAnalyticsPrivacy.HashUserId(evt.TenantId, evt.UserId),
|
||||||
|
SessionId = SearchAnalyticsPrivacy.HashSessionId(evt.TenantId, evt.SessionId),
|
||||||
|
AnswerStatus = string.IsNullOrWhiteSpace(evt.AnswerStatus)
|
||||||
|
? null
|
||||||
|
: evt.AnswerStatus.Trim().ToLowerInvariant(),
|
||||||
|
AnswerCode = string.IsNullOrWhiteSpace(evt.AnswerCode)
|
||||||
|
? null
|
||||||
|
: evt.AnswerCode.Trim().ToLowerInvariant()
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -667,11 +685,14 @@ internal record SearchAnalyticsEvent(
|
|||||||
string EventType,
|
string EventType,
|
||||||
string Query,
|
string Query,
|
||||||
string? UserId = null,
|
string? UserId = null,
|
||||||
|
string? SessionId = null,
|
||||||
string? EntityKey = null,
|
string? EntityKey = null,
|
||||||
string? Domain = null,
|
string? Domain = null,
|
||||||
int? ResultCount = null,
|
int? ResultCount = null,
|
||||||
int? Position = null,
|
int? Position = null,
|
||||||
int? DurationMs = null);
|
int? DurationMs = null,
|
||||||
|
string? AnswerStatus = null,
|
||||||
|
string? AnswerCode = null);
|
||||||
|
|
||||||
internal record SearchHistoryEntry(
|
internal record SearchHistoryEntry(
|
||||||
string HistoryId,
|
string HistoryId,
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
using System.Globalization;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
using Npgsql;
|
using Npgsql;
|
||||||
@@ -6,7 +7,8 @@ using StellaOps.AdvisoryAI.KnowledgeSearch;
|
|||||||
namespace StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
|
namespace StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Monitors search quality by analysing feedback data and zero-result queries.
|
/// Monitors search quality by analysing feedback data, zero-result queries,
|
||||||
|
/// and self-serve fallback telemetry.
|
||||||
/// Provides CRUD for search_quality_alerts and search_feedback tables.
|
/// Provides CRUD for search_quality_alerts and search_feedback tables.
|
||||||
/// Sprint: SPRINT_20260224_110 (G10-001, G10-002)
|
/// Sprint: SPRINT_20260224_110 (G10-001, G10-002)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -17,6 +19,8 @@ internal sealed class SearchQualityMonitor
|
|||||||
private const int DefaultAlertWindowDays = 7;
|
private const int DefaultAlertWindowDays = 7;
|
||||||
private const int ZeroResultAlertThreshold = 3;
|
private const int ZeroResultAlertThreshold = 3;
|
||||||
private const int NegativeFeedbackAlertThreshold = 3;
|
private const int NegativeFeedbackAlertThreshold = 3;
|
||||||
|
private const int FallbackLoopAlertThreshold = 3;
|
||||||
|
private const int AbandonedFallbackAlertThreshold = 3;
|
||||||
|
|
||||||
private readonly KnowledgeSearchOptions _options;
|
private readonly KnowledgeSearchOptions _options;
|
||||||
private readonly ILogger<SearchQualityMonitor> _logger;
|
private readonly ILogger<SearchQualityMonitor> _logger;
|
||||||
@@ -175,6 +179,32 @@ internal sealed class SearchQualityMonitor
|
|||||||
candidate.LastSeen,
|
candidate.LastSeen,
|
||||||
ct).ConfigureAwait(false);
|
ct).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var selfServeSignals = await LoadSelfServeSignalEventsAsync(tenantId, window, ct).ConfigureAwait(false);
|
||||||
|
|
||||||
|
foreach (var candidate in BuildFallbackLoopCandidates(selfServeSignals))
|
||||||
|
{
|
||||||
|
await UpsertAlertAsync(
|
||||||
|
tenantId,
|
||||||
|
alertType: "fallback_loop",
|
||||||
|
candidate.Query,
|
||||||
|
candidate.OccurrenceCount,
|
||||||
|
candidate.FirstSeen,
|
||||||
|
candidate.LastSeen,
|
||||||
|
ct).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var candidate in BuildAbandonedFallbackCandidates(selfServeSignals))
|
||||||
|
{
|
||||||
|
await UpsertAlertAsync(
|
||||||
|
tenantId,
|
||||||
|
alertType: "abandoned_fallback",
|
||||||
|
candidate.Query,
|
||||||
|
candidate.OccurrenceCount,
|
||||||
|
candidate.FirstSeen,
|
||||||
|
candidate.LastSeen,
|
||||||
|
ct).ConfigureAwait(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<IReadOnlyList<SearchQualityAlertEntry>> GetAlertsAsync(
|
public async Task<IReadOnlyList<SearchQualityAlertEntry>> GetAlertsAsync(
|
||||||
@@ -423,6 +453,9 @@ internal sealed class SearchQualityMonitor
|
|||||||
}
|
}
|
||||||
await feedbackReader.CloseAsync().ConfigureAwait(false);
|
await feedbackReader.CloseAsync().ConfigureAwait(false);
|
||||||
|
|
||||||
|
var selfServeSignals = await LoadSelfServeSignalEventsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
||||||
|
ApplySelfServeMetrics(metrics, selfServeSignals);
|
||||||
|
|
||||||
metrics.LowQualityResults = await LoadLowQualityResultsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
metrics.LowQualityResults = await LoadLowQualityResultsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
||||||
metrics.TopQueries = await LoadTopQueriesAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
metrics.TopQueries = await LoadTopQueriesAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
||||||
metrics.Trend = await LoadTrendPointsAsync(conn, tenantId, ct).ConfigureAwait(false);
|
metrics.Trend = await LoadTrendPointsAsync(conn, tenantId, ct).ConfigureAwait(false);
|
||||||
@@ -473,7 +506,8 @@ internal sealed class SearchQualityMonitor
|
|||||||
? 0d
|
? 0d
|
||||||
: (double)helpfulCount / feedbackSignals.Length * 100d;
|
: (double)helpfulCount / feedbackSignals.Length * 100d;
|
||||||
|
|
||||||
return new SearchQualityMetricsEntry
|
var selfServeSignals = BuildFallbackSelfServeSignalEvents(tenantId, window);
|
||||||
|
var metrics = new SearchQualityMetricsEntry
|
||||||
{
|
{
|
||||||
TotalSearches = totalSearches,
|
TotalSearches = totalSearches,
|
||||||
ZeroResultRate = totalSearches == 0 ? 0d : Math.Round((double)zeroResults / totalSearches * 100d, 1),
|
ZeroResultRate = totalSearches == 0 ? 0d : Math.Round((double)zeroResults / totalSearches * 100d, 1),
|
||||||
@@ -484,6 +518,10 @@ internal sealed class SearchQualityMonitor
|
|||||||
TopQueries = BuildFallbackTopQueries(tenantId, window),
|
TopQueries = BuildFallbackTopQueries(tenantId, window),
|
||||||
Trend = BuildFallbackTrendPoints(tenantId),
|
Trend = BuildFallbackTrendPoints(tenantId),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
ApplySelfServeMetrics(metrics, selfServeSignals);
|
||||||
|
|
||||||
|
return metrics;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<IReadOnlyList<SearchQualityLowQualityRow>> LoadLowQualityResultsAsync(
|
private async Task<IReadOnlyList<SearchQualityLowQualityRow>> LoadLowQualityResultsAsync(
|
||||||
@@ -758,6 +796,232 @@ internal sealed class SearchQualityMonitor
|
|||||||
return points;
|
return points;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task<IReadOnlyList<SelfServeSignalEvent>> LoadSelfServeSignalEventsAsync(
|
||||||
|
string tenantId,
|
||||||
|
TimeSpan window,
|
||||||
|
CancellationToken ct)
|
||||||
|
{
|
||||||
|
var days = Math.Max(1, (int)Math.Ceiling(window.TotalDays));
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(_options.ConnectionString))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var conn = new NpgsqlConnection(_options.ConnectionString);
|
||||||
|
await conn.OpenAsync(ct).ConfigureAwait(false);
|
||||||
|
return await LoadSelfServeSignalEventsAsync(conn, tenantId, days, ct).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogDebug(ex, "Failed to load self-serve signal events from database.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return BuildFallbackSelfServeSignalEvents(tenantId, window);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<IReadOnlyList<SelfServeSignalEvent>> LoadSelfServeSignalEventsAsync(
|
||||||
|
NpgsqlConnection conn,
|
||||||
|
string tenantId,
|
||||||
|
int days,
|
||||||
|
CancellationToken ct)
|
||||||
|
{
|
||||||
|
var events = new List<SelfServeSignalEvent>();
|
||||||
|
await using var cmd = new NpgsqlCommand(@"
|
||||||
|
SELECT
|
||||||
|
COALESCE(session_id, ''),
|
||||||
|
event_type,
|
||||||
|
query,
|
||||||
|
answer_status,
|
||||||
|
created_at,
|
||||||
|
event_id::text
|
||||||
|
FROM advisoryai.search_events
|
||||||
|
WHERE tenant_id = @tenant_id
|
||||||
|
AND created_at > now() - make_interval(days => @days)
|
||||||
|
AND event_type IN ('answer_frame', 'reformulation', 'rescue_action', 'click')
|
||||||
|
ORDER BY created_at ASC, event_id ASC", conn);
|
||||||
|
cmd.Parameters.AddWithValue("tenant_id", tenantId);
|
||||||
|
cmd.Parameters.AddWithValue("days", days);
|
||||||
|
|
||||||
|
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||||
|
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||||
|
{
|
||||||
|
events.Add(new SelfServeSignalEvent(
|
||||||
|
SessionId: reader.IsDBNull(0) ? null : NullIfEmpty(reader.GetString(0)),
|
||||||
|
EventType: reader.GetString(1),
|
||||||
|
Query: reader.GetString(2),
|
||||||
|
AnswerStatus: reader.IsDBNull(3) ? null : reader.GetString(3),
|
||||||
|
CreatedAt: new DateTimeOffset(reader.GetDateTime(4), TimeSpan.Zero),
|
||||||
|
OrderKey: reader.GetString(5)));
|
||||||
|
}
|
||||||
|
|
||||||
|
return events;
|
||||||
|
}
|
||||||
|
|
||||||
|
private IReadOnlyList<SelfServeSignalEvent> BuildFallbackSelfServeSignalEvents(string tenantId, TimeSpan window)
|
||||||
|
{
|
||||||
|
return _analyticsService.GetFallbackEventsSnapshot(tenantId, window)
|
||||||
|
.Where(static item =>
|
||||||
|
item.Event.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
item.Event.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
item.Event.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
item.Event.EventType.Equals("click", StringComparison.OrdinalIgnoreCase))
|
||||||
|
.Select((item, index) => new SelfServeSignalEvent(
|
||||||
|
SessionId: item.Event.SessionId,
|
||||||
|
EventType: item.Event.EventType,
|
||||||
|
Query: item.Event.Query,
|
||||||
|
AnswerStatus: item.Event.AnswerStatus,
|
||||||
|
CreatedAt: item.RecordedAt,
|
||||||
|
OrderKey: index.ToString("D8", CultureInfo.InvariantCulture)))
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ApplySelfServeMetrics(SearchQualityMetricsEntry metrics, IReadOnlyList<SelfServeSignalEvent> signals)
|
||||||
|
{
|
||||||
|
var totalAnswerFrames = signals.Count(static evt =>
|
||||||
|
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase));
|
||||||
|
var fallbackAnswers = signals.Count(IsFallbackAnswerEvent);
|
||||||
|
var clarifyAnswers = signals.Count(static evt =>
|
||||||
|
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) &&
|
||||||
|
string.Equals(evt.AnswerStatus, "clarify", StringComparison.OrdinalIgnoreCase));
|
||||||
|
var insufficientAnswers = signals.Count(static evt =>
|
||||||
|
evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase) &&
|
||||||
|
string.Equals(evt.AnswerStatus, "insufficient", StringComparison.OrdinalIgnoreCase));
|
||||||
|
|
||||||
|
metrics.FallbackAnswerRate = totalAnswerFrames == 0
|
||||||
|
? 0d
|
||||||
|
: Math.Round((double)fallbackAnswers / totalAnswerFrames * 100d, 1);
|
||||||
|
metrics.ClarifyRate = totalAnswerFrames == 0
|
||||||
|
? 0d
|
||||||
|
: Math.Round((double)clarifyAnswers / totalAnswerFrames * 100d, 1);
|
||||||
|
metrics.InsufficientRate = totalAnswerFrames == 0
|
||||||
|
? 0d
|
||||||
|
: Math.Round((double)insufficientAnswers / totalAnswerFrames * 100d, 1);
|
||||||
|
metrics.ReformulationCount = signals.Count(static evt =>
|
||||||
|
evt.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase));
|
||||||
|
metrics.RescueActionCount = signals.Count(static evt =>
|
||||||
|
evt.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase));
|
||||||
|
metrics.AbandonedFallbackCount = GetAbandonedFallbackSessions(signals).Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IReadOnlyList<AlertCandidate> BuildFallbackLoopCandidates(IReadOnlyList<SelfServeSignalEvent> events)
|
||||||
|
{
|
||||||
|
var rawCandidates = new List<AlertCandidate>();
|
||||||
|
foreach (var group in events
|
||||||
|
.Where(static evt => !string.IsNullOrWhiteSpace(evt.SessionId))
|
||||||
|
.GroupBy(static evt => evt.SessionId!, StringComparer.Ordinal))
|
||||||
|
{
|
||||||
|
var ordered = OrderSessionEvents(group);
|
||||||
|
var fallbackFrames = ordered.Where(IsFallbackAnswerEvent).ToArray();
|
||||||
|
if (fallbackFrames.Length < FallbackLoopAlertThreshold)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var first = fallbackFrames[0];
|
||||||
|
var last = fallbackFrames[^1];
|
||||||
|
rawCandidates.Add(new AlertCandidate(last.Query, fallbackFrames.Length, first.CreatedAt, last.CreatedAt));
|
||||||
|
}
|
||||||
|
|
||||||
|
return rawCandidates
|
||||||
|
.GroupBy(static candidate => candidate.Query, StringComparer.OrdinalIgnoreCase)
|
||||||
|
.Select(group => new AlertCandidate(
|
||||||
|
group.Key,
|
||||||
|
group.Sum(static candidate => candidate.OccurrenceCount),
|
||||||
|
group.Min(static candidate => candidate.FirstSeen),
|
||||||
|
group.Max(static candidate => candidate.LastSeen)))
|
||||||
|
.OrderByDescending(static candidate => candidate.OccurrenceCount)
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IReadOnlyList<AlertCandidate> BuildAbandonedFallbackCandidates(IReadOnlyList<SelfServeSignalEvent> events)
|
||||||
|
{
|
||||||
|
return GetAbandonedFallbackSessions(events)
|
||||||
|
.GroupBy(static candidate => candidate.Query, StringComparer.OrdinalIgnoreCase)
|
||||||
|
.Select(group => new AlertCandidate(
|
||||||
|
group.Key,
|
||||||
|
group.Sum(static candidate => candidate.OccurrenceCount),
|
||||||
|
group.Min(static candidate => candidate.FirstSeen),
|
||||||
|
group.Max(static candidate => candidate.LastSeen)))
|
||||||
|
.Where(static candidate => candidate.OccurrenceCount >= AbandonedFallbackAlertThreshold)
|
||||||
|
.OrderByDescending(static candidate => candidate.OccurrenceCount)
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static IReadOnlyList<AlertCandidate> GetAbandonedFallbackSessions(IReadOnlyList<SelfServeSignalEvent> events)
|
||||||
|
{
|
||||||
|
var candidates = new List<AlertCandidate>();
|
||||||
|
foreach (var group in events
|
||||||
|
.Where(static evt => !string.IsNullOrWhiteSpace(evt.SessionId))
|
||||||
|
.GroupBy(static evt => evt.SessionId!, StringComparer.Ordinal))
|
||||||
|
{
|
||||||
|
var ordered = OrderSessionEvents(group);
|
||||||
|
var lastFallbackIndex = -1;
|
||||||
|
for (var index = ordered.Length - 1; index >= 0; index--)
|
||||||
|
{
|
||||||
|
if (IsFallbackAnswerEvent(ordered[index]))
|
||||||
|
{
|
||||||
|
lastFallbackIndex = index;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lastFallbackIndex < 0)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastFallback = ordered[lastFallbackIndex];
|
||||||
|
var recovered = false;
|
||||||
|
for (var index = lastFallbackIndex + 1; index < ordered.Length; index++)
|
||||||
|
{
|
||||||
|
if (IsRecoveryEvent(ordered[index]))
|
||||||
|
{
|
||||||
|
recovered = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (recovered)
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates.Add(new AlertCandidate(lastFallback.Query, 1, lastFallback.CreatedAt, lastFallback.CreatedAt));
|
||||||
|
}
|
||||||
|
|
||||||
|
return candidates;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsFallbackAnswerEvent(SelfServeSignalEvent evt)
|
||||||
|
{
|
||||||
|
return evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase)
|
||||||
|
&& (string.Equals(evt.AnswerStatus, "clarify", StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| string.Equals(evt.AnswerStatus, "insufficient", StringComparison.OrdinalIgnoreCase));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsRecoveryEvent(SelfServeSignalEvent evt)
|
||||||
|
{
|
||||||
|
return evt.EventType.Equals("click", StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| evt.EventType.Equals("reformulation", StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| evt.EventType.Equals("rescue_action", StringComparison.OrdinalIgnoreCase)
|
||||||
|
|| (evt.EventType.Equals("answer_frame", StringComparison.OrdinalIgnoreCase)
|
||||||
|
&& string.Equals(evt.AnswerStatus, "grounded", StringComparison.OrdinalIgnoreCase));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SelfServeSignalEvent[] OrderSessionEvents(IEnumerable<SelfServeSignalEvent> events)
|
||||||
|
{
|
||||||
|
return events
|
||||||
|
.OrderBy(static evt => evt.CreatedAt)
|
||||||
|
.ThenBy(static evt => evt.OrderKey, StringComparer.Ordinal)
|
||||||
|
.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string? NullIfEmpty(string value)
|
||||||
|
{
|
||||||
|
return string.IsNullOrWhiteSpace(value) ? null : value;
|
||||||
|
}
|
||||||
|
|
||||||
private async Task<IReadOnlyList<AlertCandidate>> LoadZeroResultCandidatesAsync(
|
private async Task<IReadOnlyList<AlertCandidate>> LoadZeroResultCandidatesAsync(
|
||||||
string tenantId,
|
string tenantId,
|
||||||
TimeSpan window,
|
TimeSpan window,
|
||||||
@@ -1170,12 +1434,26 @@ internal sealed class SearchQualityMetricsEntry
|
|||||||
public double ZeroResultRate { get; set; }
|
public double ZeroResultRate { get; set; }
|
||||||
public double AvgResultCount { get; set; }
|
public double AvgResultCount { get; set; }
|
||||||
public double FeedbackScore { get; set; }
|
public double FeedbackScore { get; set; }
|
||||||
|
public double FallbackAnswerRate { get; set; }
|
||||||
|
public double ClarifyRate { get; set; }
|
||||||
|
public double InsufficientRate { get; set; }
|
||||||
|
public int ReformulationCount { get; set; }
|
||||||
|
public int RescueActionCount { get; set; }
|
||||||
|
public int AbandonedFallbackCount { get; set; }
|
||||||
public string Period { get; set; } = "7d";
|
public string Period { get; set; } = "7d";
|
||||||
public IReadOnlyList<SearchQualityLowQualityRow> LowQualityResults { get; set; } = [];
|
public IReadOnlyList<SearchQualityLowQualityRow> LowQualityResults { get; set; } = [];
|
||||||
public IReadOnlyList<SearchQualityTopQueryRow> TopQueries { get; set; } = [];
|
public IReadOnlyList<SearchQualityTopQueryRow> TopQueries { get; set; } = [];
|
||||||
public IReadOnlyList<SearchQualityTrendPoint> Trend { get; set; } = [];
|
public IReadOnlyList<SearchQualityTrendPoint> Trend { get; set; } = [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal sealed record SelfServeSignalEvent(
|
||||||
|
string? SessionId,
|
||||||
|
string EventType,
|
||||||
|
string Query,
|
||||||
|
string? AnswerStatus,
|
||||||
|
DateTimeOffset CreatedAt,
|
||||||
|
string OrderKey);
|
||||||
|
|
||||||
internal sealed class SearchQualityLowQualityRow
|
internal sealed class SearchQualityLowQualityRow
|
||||||
{
|
{
|
||||||
public string EntityKey { get; set; } = string.Empty;
|
public string EntityKey { get; set; } = string.Empty;
|
||||||
|
|||||||
@@ -107,18 +107,23 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|||||||
return EmptyResponse(string.Empty, request.K, "empty");
|
return EmptyResponse(string.Empty, request.K, "empty");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (query.Length > _unifiedOptions.MaxQueryLength)
|
|
||||||
{
|
|
||||||
return EmptyResponse(query, request.K, "query_too_long", request.Ambient);
|
|
||||||
}
|
|
||||||
|
|
||||||
var tenantId = request.Filters?.Tenant ?? "global";
|
var tenantId = request.Filters?.Tenant ?? "global";
|
||||||
var userId = request.Filters?.UserId ?? "anonymous";
|
var userId = request.Filters?.UserId ?? "anonymous";
|
||||||
|
|
||||||
|
if (query.Length > _unifiedOptions.MaxQueryLength)
|
||||||
|
{
|
||||||
|
var earlyResponse = EmptyResponse(query, request.K, "query_too_long", request.Ambient);
|
||||||
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
|
||||||
|
return earlyResponse;
|
||||||
|
}
|
||||||
|
|
||||||
var tenantFlags = ResolveTenantFeatureFlags(tenantId);
|
var tenantFlags = ResolveTenantFeatureFlags(tenantId);
|
||||||
|
|
||||||
if (!_options.Enabled || !IsSearchEnabledForTenant(tenantFlags) || string.IsNullOrWhiteSpace(_options.ConnectionString))
|
if (!_options.Enabled || !IsSearchEnabledForTenant(tenantFlags) || string.IsNullOrWhiteSpace(_options.ConnectionString))
|
||||||
{
|
{
|
||||||
return EmptyResponse(query, request.K, "disabled", request.Ambient);
|
var earlyResponse = EmptyResponse(query, request.K, "disabled", request.Ambient);
|
||||||
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan: null, earlyResponse, cancellationToken).ConfigureAwait(false);
|
||||||
|
return earlyResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (request.Ambient?.ResetSession == true &&
|
if (request.Ambient?.ResetSession == true &&
|
||||||
@@ -321,6 +326,7 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|||||||
refinements,
|
refinements,
|
||||||
contextAnswer);
|
contextAnswer);
|
||||||
|
|
||||||
|
await RecordAnswerFrameAnalyticsAsync(tenantId, userId, request, plan, response, cancellationToken).ConfigureAwait(false);
|
||||||
EmitTelemetry(plan, response, tenantId);
|
EmitTelemetry(plan, response, tenantId);
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
@@ -1858,4 +1864,40 @@ internal sealed class UnifiedSearchService : IUnifiedSearchService
|
|||||||
HasSuggestions: response.Suggestions is { Count: > 0 },
|
HasSuggestions: response.Suggestions is { Count: > 0 },
|
||||||
HasRefinements: response.Refinements is { Count: > 0 }));
|
HasRefinements: response.Refinements is { Count: > 0 }));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task RecordAnswerFrameAnalyticsAsync(
|
||||||
|
string tenantId,
|
||||||
|
string userId,
|
||||||
|
UnifiedSearchRequest request,
|
||||||
|
QueryPlan? plan,
|
||||||
|
UnifiedSearchResponse response,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(response.Query))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var contextAnswer = response.ContextAnswer;
|
||||||
|
if (contextAnswer is null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var domain = ResolveContextDomain(plan, response.Cards, request.Ambient)
|
||||||
|
?? response.Cards.FirstOrDefault()?.Domain
|
||||||
|
?? request.Filters?.Domains?.FirstOrDefault();
|
||||||
|
|
||||||
|
await _analyticsService.RecordEventAsync(new SearchAnalyticsEvent(
|
||||||
|
TenantId: tenantId,
|
||||||
|
EventType: "answer_frame",
|
||||||
|
Query: response.Query,
|
||||||
|
UserId: userId,
|
||||||
|
SessionId: request.Ambient?.SessionId,
|
||||||
|
Domain: domain,
|
||||||
|
ResultCount: response.Cards.Count,
|
||||||
|
DurationMs: (int)Math.Min(int.MaxValue, Math.Max(0, response.Diagnostics.DurationMs)),
|
||||||
|
AnswerStatus: contextAnswer.Status,
|
||||||
|
AnswerCode: contextAnswer.Code), cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1511,6 +1511,12 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
|||||||
"zero-result rate must be computed from raw zero_result analytics events");
|
"zero-result rate must be computed from raw zero_result analytics events");
|
||||||
metrics.AvgResultCount.Should().BeApproximately(2.0, 0.2);
|
metrics.AvgResultCount.Should().BeApproximately(2.0, 0.2);
|
||||||
metrics.FeedbackScore.Should().BeApproximately(50.0, 0.2);
|
metrics.FeedbackScore.Should().BeApproximately(50.0, 0.2);
|
||||||
|
metrics.FallbackAnswerRate.Should().Be(0);
|
||||||
|
metrics.ClarifyRate.Should().Be(0);
|
||||||
|
metrics.InsufficientRate.Should().Be(0);
|
||||||
|
metrics.ReformulationCount.Should().Be(0);
|
||||||
|
metrics.RescueActionCount.Should().Be(0);
|
||||||
|
metrics.AbandonedFallbackCount.Should().Be(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -1579,6 +1585,74 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
|||||||
.Should().BeTrue("low-quality rows should include repeated negative feedback entities");
|
.Should().BeTrue("low-quality rows should include repeated negative feedback entities");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task G10_SelfServeMetrics_IncludeFallbackReformulationAndRescueSignals()
|
||||||
|
{
|
||||||
|
var tenant = $"selfserve-metrics-{Guid.NewGuid():N}";
|
||||||
|
using var writer = CreateAuthenticatedClient(tenant);
|
||||||
|
using var admin = CreateAdminClient(tenant);
|
||||||
|
|
||||||
|
var analyticsResponse = await writer.PostAsJsonAsync("/v1/advisory-ai/search/analytics", new SearchAnalyticsApiRequest
|
||||||
|
{
|
||||||
|
Events =
|
||||||
|
[
|
||||||
|
new SearchAnalyticsApiEvent
|
||||||
|
{
|
||||||
|
EventType = "query",
|
||||||
|
Query = "database connectivity",
|
||||||
|
ResultCount = 0
|
||||||
|
},
|
||||||
|
new SearchAnalyticsApiEvent
|
||||||
|
{
|
||||||
|
EventType = "answer_frame",
|
||||||
|
Query = "database connectivity",
|
||||||
|
SessionId = "session-selfserve-1",
|
||||||
|
Domain = "knowledge",
|
||||||
|
AnswerStatus = "clarify",
|
||||||
|
AnswerCode = "query_needs_scope"
|
||||||
|
},
|
||||||
|
new SearchAnalyticsApiEvent
|
||||||
|
{
|
||||||
|
EventType = "reformulation",
|
||||||
|
Query = "postgresql connectivity",
|
||||||
|
SessionId = "session-selfserve-1",
|
||||||
|
Domain = "knowledge"
|
||||||
|
},
|
||||||
|
new SearchAnalyticsApiEvent
|
||||||
|
{
|
||||||
|
EventType = "answer_frame",
|
||||||
|
Query = "postgresql connectivity",
|
||||||
|
SessionId = "session-selfserve-1",
|
||||||
|
Domain = "knowledge",
|
||||||
|
AnswerStatus = "insufficient",
|
||||||
|
AnswerCode = "no_grounded_evidence"
|
||||||
|
},
|
||||||
|
new SearchAnalyticsApiEvent
|
||||||
|
{
|
||||||
|
EventType = "rescue_action",
|
||||||
|
Query = "postgresql connectivity",
|
||||||
|
SessionId = "session-selfserve-1",
|
||||||
|
Domain = "knowledge"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
analyticsResponse.StatusCode.Should().Be(HttpStatusCode.NoContent);
|
||||||
|
|
||||||
|
var metricsResponse = await admin.GetAsync("/v1/advisory-ai/search/quality/metrics?period=7d");
|
||||||
|
metricsResponse.StatusCode.Should().Be(HttpStatusCode.OK);
|
||||||
|
|
||||||
|
var metrics = await metricsResponse.Content.ReadFromJsonAsync<SearchQualityMetricsDto>();
|
||||||
|
metrics.Should().NotBeNull();
|
||||||
|
metrics!.FallbackAnswerRate.Should().BeApproximately(100.0, 0.2);
|
||||||
|
metrics.ClarifyRate.Should().BeApproximately(50.0, 0.2);
|
||||||
|
metrics.InsufficientRate.Should().BeApproximately(50.0, 0.2);
|
||||||
|
metrics.ReformulationCount.Should().Be(1);
|
||||||
|
metrics.RescueActionCount.Should().Be(1);
|
||||||
|
metrics.AbandonedFallbackCount.Should().Be(0,
|
||||||
|
"a rescue action in the same session means the fallback flow was not abandoned");
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task G10_AnalyticsEndpoint_ValidBatch_ReturnsNoContent()
|
public async Task G10_AnalyticsEndpoint_ValidBatch_ReturnsNoContent()
|
||||||
{
|
{
|
||||||
@@ -1624,6 +1698,7 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
|||||||
EventType: "query",
|
EventType: "query",
|
||||||
Query: query,
|
Query: query,
|
||||||
UserId: userId,
|
UserId: userId,
|
||||||
|
SessionId: "session-privacy-1",
|
||||||
ResultCount: 1));
|
ResultCount: 1));
|
||||||
|
|
||||||
var events = analyticsService.GetFallbackEventsSnapshot(tenant, TimeSpan.FromMinutes(1));
|
var events = analyticsService.GetFallbackEventsSnapshot(tenant, TimeSpan.FromMinutes(1));
|
||||||
@@ -1634,6 +1709,7 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
|||||||
stored.Query.Should().NotContain("alice", "raw query text must not be stored in analytics events");
|
stored.Query.Should().NotContain("alice", "raw query text must not be stored in analytics events");
|
||||||
stored.UserId.Should().Be(SearchAnalyticsPrivacy.HashUserId(tenant, userId));
|
stored.UserId.Should().Be(SearchAnalyticsPrivacy.HashUserId(tenant, userId));
|
||||||
stored.UserId.Should().NotBe(userId, "user identifiers must be pseudonymized before persistence");
|
stored.UserId.Should().NotBe(userId, "user identifiers must be pseudonymized before persistence");
|
||||||
|
stored.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId(tenant, "session-privacy-1"));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -1862,6 +1938,76 @@ public sealed class UnifiedSearchSprintIntegrationTests : IDisposable
|
|||||||
.Should().BeTrue("five repeated zero-result events should create a zero_result quality alert");
|
.Should().BeTrue("five repeated zero-result events should create a zero_result quality alert");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task G10_FallbackLoopSessions_CreateQualityAlert()
|
||||||
|
{
|
||||||
|
var tenant = $"fallback-loop-{Guid.NewGuid():N}";
|
||||||
|
using var scope = _factory.Services.CreateScope();
|
||||||
|
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
|
||||||
|
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
|
||||||
|
|
||||||
|
await analytics.RecordEventsAsync(
|
||||||
|
[
|
||||||
|
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate", SessionId: "sess-loop-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
|
||||||
|
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate release", SessionId: "sess-loop-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
|
||||||
|
new SearchAnalyticsEvent(tenant, "answer_frame", "policy gate release production", SessionId: "sess-loop-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence")
|
||||||
|
]);
|
||||||
|
|
||||||
|
await monitor.RefreshAlertsAsync(tenant);
|
||||||
|
var alerts = await monitor.GetAlertsAsync(tenant, status: "open", alertType: "fallback_loop");
|
||||||
|
|
||||||
|
var expectedQueryHash = SearchAnalyticsPrivacy.HashQuery("policy gate release production");
|
||||||
|
alerts.Any(alert => alert.Query.Equals(expectedQueryHash, StringComparison.OrdinalIgnoreCase))
|
||||||
|
.Should().BeTrue("three fallback answer frames in the same session should create a fallback_loop alert");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task G10_AbandonedFallbackSessions_CreateQualityAlert()
|
||||||
|
{
|
||||||
|
var tenant = $"abandoned-fallback-{Guid.NewGuid():N}";
|
||||||
|
using var scope = _factory.Services.CreateScope();
|
||||||
|
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
|
||||||
|
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
|
||||||
|
|
||||||
|
await analytics.RecordEventsAsync(
|
||||||
|
[
|
||||||
|
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
|
||||||
|
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-2", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
|
||||||
|
new SearchAnalyticsEvent(tenant, "answer_frame", "unknown control token", SessionId: "sess-abandon-3", AnswerStatus: "clarify", AnswerCode: "query_needs_scope")
|
||||||
|
]);
|
||||||
|
|
||||||
|
await monitor.RefreshAlertsAsync(tenant);
|
||||||
|
var alerts = await monitor.GetAlertsAsync(tenant, status: "open", alertType: "abandoned_fallback");
|
||||||
|
|
||||||
|
var expectedQueryHash = SearchAnalyticsPrivacy.HashQuery("unknown control token");
|
||||||
|
alerts.Any(alert => alert.Query.Equals(expectedQueryHash, StringComparison.OrdinalIgnoreCase))
|
||||||
|
.Should().BeTrue("repeated sessions that end in fallback without recovery should create an abandoned_fallback alert");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task G10_RecoveredFallbackSessions_DoNotCountAsAbandoned()
|
||||||
|
{
|
||||||
|
var tenant = $"recovered-fallback-{Guid.NewGuid():N}";
|
||||||
|
using var scope = _factory.Services.CreateScope();
|
||||||
|
var analytics = scope.ServiceProvider.GetRequiredService<SearchAnalyticsService>();
|
||||||
|
var monitor = scope.ServiceProvider.GetRequiredService<SearchQualityMonitor>();
|
||||||
|
|
||||||
|
await analytics.RecordEventsAsync(
|
||||||
|
[
|
||||||
|
new SearchAnalyticsEvent(tenant, "answer_frame", "database connectivity", SessionId: "sess-recovered-1", AnswerStatus: "clarify", AnswerCode: "query_needs_scope"),
|
||||||
|
new SearchAnalyticsEvent(tenant, "reformulation", "postgresql connectivity", SessionId: "sess-recovered-1"),
|
||||||
|
new SearchAnalyticsEvent(tenant, "answer_frame", "postgresql connectivity", SessionId: "sess-recovered-1", AnswerStatus: "insufficient", AnswerCode: "no_grounded_evidence"),
|
||||||
|
new SearchAnalyticsEvent(tenant, "rescue_action", "postgresql connectivity", SessionId: "sess-recovered-1")
|
||||||
|
]);
|
||||||
|
|
||||||
|
var metrics = await monitor.GetMetricsAsync(tenant, "7d");
|
||||||
|
|
||||||
|
metrics.ReformulationCount.Should().Be(1);
|
||||||
|
metrics.RescueActionCount.Should().Be(1);
|
||||||
|
metrics.AbandonedFallbackCount.Should().Be(0,
|
||||||
|
"a session with a reformulation and rescue action after fallback is a recovery path, not an abandonment");
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task G10_NegativeFeedbackBurst_CreatesHighNegativeFeedbackAlert()
|
public async Task G10_NegativeFeedbackBurst_CreatesHighNegativeFeedbackAlert()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -22,4 +22,5 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
|
|||||||
| SPRINT_20260224_G1-004-BENCH | DONE | Semantic recall benchmark: 13 tests in `SemanticRecallBenchmarkTests.cs`, 48-query fixture (`semantic-recall-benchmark.json`), `SemanticRecallBenchmarkStore` (33 chunks), `SemanticSimulationEncoder` (40+ semantic groups). Semantic strictly outperforms hash on synonym queries. |
|
| SPRINT_20260224_G1-004-BENCH | DONE | Semantic recall benchmark: 13 tests in `SemanticRecallBenchmarkTests.cs`, 48-query fixture (`semantic-recall-benchmark.json`), `SemanticRecallBenchmarkStore` (33 chunks), `SemanticSimulationEncoder` (40+ semantic groups). Semantic strictly outperforms hash on synonym queries. |
|
||||||
|
|
||||||
| AI-SELF-005 | DONE | Integration coverage now asserts grounded, clarify, and insufficient contextual-answer states through the real endpoint contract. |
|
| AI-SELF-005 | DONE | Integration coverage now asserts grounded, clarify, and insufficient contextual-answer states through the real endpoint contract. |
|
||||||
|
| AI-SELF-004 | DONE | Targeted telemetry coverage now verifies fallback metrics, recovered-session suppression of abandoned counts, session hashing, and self-serve alert generation with xUnit v3-compatible runner commands. |
|
||||||
| AI-SELF-006 | DONE | Verification includes a real local corpus rebuild and a live query assertion, not only test doubles. |
|
| AI-SELF-006 | DONE | Verification includes a real local corpus rebuild and a live query assertion, not only test doubles. |
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ using StellaOps.AdvisoryAI.UnifiedSearch.Analytics;
|
|||||||
using StellaOps.AdvisoryAI.UnifiedSearch.QueryUnderstanding;
|
using StellaOps.AdvisoryAI.UnifiedSearch.QueryUnderstanding;
|
||||||
using StellaOps.AdvisoryAI.UnifiedSearch.Synthesis;
|
using StellaOps.AdvisoryAI.UnifiedSearch.Synthesis;
|
||||||
using StellaOps.AdvisoryAI.Vectorization;
|
using StellaOps.AdvisoryAI.Vectorization;
|
||||||
|
using System.Linq;
|
||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using Xunit;
|
using Xunit;
|
||||||
|
|
||||||
@@ -124,6 +125,101 @@ public sealed class UnifiedSearchServiceTests
|
|||||||
result.ContextAnswer.Questions!.Should().OnlyContain(question => question.Kind == "recover");
|
result.ContextAnswer.Questions!.Should().OnlyContain(question => question.Kind == "recover");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task SearchAsync_records_answer_frame_analytics_for_clarify_state()
|
||||||
|
{
|
||||||
|
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
|
||||||
|
{
|
||||||
|
Enabled = true,
|
||||||
|
ConnectionString = "Host=localhost;Database=test"
|
||||||
|
});
|
||||||
|
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
|
||||||
|
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
|
||||||
|
var service = CreateService(
|
||||||
|
storeMock: CreateEmptyStoreMock(),
|
||||||
|
analyticsService: analyticsService,
|
||||||
|
qualityMonitor: qualityMonitor);
|
||||||
|
|
||||||
|
var result = await service.SearchAsync(
|
||||||
|
new UnifiedSearchRequest(
|
||||||
|
"status",
|
||||||
|
Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" },
|
||||||
|
Ambient: new AmbientContext
|
||||||
|
{
|
||||||
|
CurrentRoute = "/ops/operations/doctor",
|
||||||
|
SessionId = "session-123"
|
||||||
|
}),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
result.ContextAnswer.Should().NotBeNull();
|
||||||
|
result.ContextAnswer!.Status.Should().Be("clarify");
|
||||||
|
|
||||||
|
var events = analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1));
|
||||||
|
events.Should().ContainSingle();
|
||||||
|
var answerFrame = events.Single().Event;
|
||||||
|
answerFrame.EventType.Should().Be("answer_frame");
|
||||||
|
answerFrame.AnswerStatus.Should().Be("clarify");
|
||||||
|
answerFrame.AnswerCode.Should().Be(result.ContextAnswer.Code);
|
||||||
|
answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-123"));
|
||||||
|
answerFrame.Query.Should().Be(SearchAnalyticsPrivacy.HashQuery("status"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task SearchAsync_records_answer_frame_analytics_for_grounded_state()
|
||||||
|
{
|
||||||
|
var analyticsOptions = Options.Create(new KnowledgeSearchOptions
|
||||||
|
{
|
||||||
|
Enabled = true,
|
||||||
|
ConnectionString = "Host=localhost;Database=test"
|
||||||
|
});
|
||||||
|
var analyticsService = new SearchAnalyticsService(analyticsOptions, NullLogger<SearchAnalyticsService>.Instance);
|
||||||
|
var qualityMonitor = new SearchQualityMonitor(analyticsOptions, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
|
||||||
|
|
||||||
|
var doctorRow = MakeRow(
|
||||||
|
"chunk-doctor-analytics",
|
||||||
|
"doctor_check",
|
||||||
|
"PostgreSQL connectivity",
|
||||||
|
JsonDocument.Parse("{\"domain\":\"knowledge\",\"checkCode\":\"check.core.db.connectivity\"}"));
|
||||||
|
|
||||||
|
var storeMock = new Mock<IKnowledgeSearchStore>();
|
||||||
|
storeMock.Setup(s => s.SearchFtsAsync(
|
||||||
|
It.IsAny<string>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||||
|
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>(), It.IsAny<string?>()))
|
||||||
|
.ReturnsAsync(new List<KnowledgeChunkRow> { doctorRow });
|
||||||
|
storeMock.Setup(s => s.LoadVectorCandidatesAsync(
|
||||||
|
It.IsAny<float[]>(), It.IsAny<KnowledgeSearchFilter?>(), It.IsAny<int>(),
|
||||||
|
It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
|
||||||
|
.ReturnsAsync([]);
|
||||||
|
|
||||||
|
var service = CreateService(
|
||||||
|
storeMock: storeMock,
|
||||||
|
analyticsService: analyticsService,
|
||||||
|
qualityMonitor: qualityMonitor);
|
||||||
|
|
||||||
|
var result = await service.SearchAsync(
|
||||||
|
new UnifiedSearchRequest(
|
||||||
|
"database connectivity",
|
||||||
|
Filters: new UnifiedSearchFilter { Tenant = "tenant-telemetry", UserId = "user-1" },
|
||||||
|
Ambient: new AmbientContext
|
||||||
|
{
|
||||||
|
CurrentRoute = "/ops/operations/doctor",
|
||||||
|
SessionId = "session-456"
|
||||||
|
}),
|
||||||
|
CancellationToken.None);
|
||||||
|
|
||||||
|
result.ContextAnswer.Should().NotBeNull();
|
||||||
|
result.ContextAnswer!.Status.Should().Be("grounded");
|
||||||
|
|
||||||
|
var events = analyticsService.GetFallbackEventsSnapshot("tenant-telemetry", TimeSpan.FromMinutes(1));
|
||||||
|
events.Should().ContainSingle();
|
||||||
|
var answerFrame = events.Single().Event;
|
||||||
|
answerFrame.EventType.Should().Be("answer_frame");
|
||||||
|
answerFrame.AnswerStatus.Should().Be("grounded");
|
||||||
|
answerFrame.AnswerCode.Should().Be("retrieved_evidence");
|
||||||
|
answerFrame.ResultCount.Should().Be(1);
|
||||||
|
answerFrame.SessionId.Should().Be(SearchAnalyticsPrivacy.HashSessionId("tenant-telemetry", "session-456"));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task SearchAsync_returns_empty_when_tenant_feature_flag_disables_search()
|
public async Task SearchAsync_returns_empty_when_tenant_feature_flag_disables_search()
|
||||||
{
|
{
|
||||||
@@ -827,7 +923,9 @@ public sealed class UnifiedSearchServiceTests
|
|||||||
private static UnifiedSearchService CreateService(
|
private static UnifiedSearchService CreateService(
|
||||||
bool enabled = true,
|
bool enabled = true,
|
||||||
Mock<IKnowledgeSearchStore>? storeMock = null,
|
Mock<IKnowledgeSearchStore>? storeMock = null,
|
||||||
UnifiedSearchOptions? unifiedOptions = null)
|
UnifiedSearchOptions? unifiedOptions = null,
|
||||||
|
SearchAnalyticsService? analyticsService = null,
|
||||||
|
SearchQualityMonitor? qualityMonitor = null)
|
||||||
{
|
{
|
||||||
var options = Options.Create(new KnowledgeSearchOptions
|
var options = Options.Create(new KnowledgeSearchOptions
|
||||||
{
|
{
|
||||||
@@ -855,8 +953,8 @@ public sealed class UnifiedSearchServiceTests
|
|||||||
var weightCalculator = new DomainWeightCalculator(extractor, classifier, options);
|
var weightCalculator = new DomainWeightCalculator(extractor, classifier, options);
|
||||||
var planBuilder = new QueryPlanBuilder(extractor, classifier, weightCalculator);
|
var planBuilder = new QueryPlanBuilder(extractor, classifier, weightCalculator);
|
||||||
var synthesisEngine = new SynthesisTemplateEngine();
|
var synthesisEngine = new SynthesisTemplateEngine();
|
||||||
var analyticsService = new SearchAnalyticsService(options, NullLogger<SearchAnalyticsService>.Instance);
|
analyticsService ??= new SearchAnalyticsService(options, NullLogger<SearchAnalyticsService>.Instance);
|
||||||
var qualityMonitor = new SearchQualityMonitor(options, NullLogger<SearchQualityMonitor>.Instance);
|
qualityMonitor ??= new SearchQualityMonitor(options, NullLogger<SearchQualityMonitor>.Instance, analyticsService);
|
||||||
var entityAliasService = new Mock<IEntityAliasService>();
|
var entityAliasService = new Mock<IEntityAliasService>();
|
||||||
entityAliasService.Setup(s => s.ResolveAliasesAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
entityAliasService.Setup(s => s.ResolveAliasesAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||||
.ReturnsAsync(Array.Empty<(string EntityKey, string EntityType)>());
|
.ReturnsAsync(Array.Empty<(string EntityKey, string EntityType)>());
|
||||||
|
|||||||
Reference in New Issue
Block a user