From e11c0a6b59f2efa6f3bb46f2f14eac408bfa87f5 Mon Sep 17 00:00:00 2001 From: master <> Date: Sat, 7 Mar 2026 21:49:41 +0200 Subject: [PATCH] Add live search readiness and telemetry-off e2e coverage --- ...307_034_FE_live_search_readiness_matrix.md | 7 +- src/AdvisoryAI/__Tests/INFRASTRUCTURE.md | 4 +- ...ch-contextual-suggestions.live.e2e.spec.ts | 19 +++-- ...fied-search-experience-quality.e2e.spec.ts | 71 +++++++++++++++++++ 4 files changed, 86 insertions(+), 15 deletions(-) diff --git a/docs/implplan/SPRINT_20260307_034_FE_live_search_readiness_matrix.md b/docs/implplan/SPRINT_20260307_034_FE_live_search_readiness_matrix.md index e2817d69a..f6d48255f 100644 --- a/docs/implplan/SPRINT_20260307_034_FE_live_search_readiness_matrix.md +++ b/docs/implplan/SPRINT_20260307_034_FE_live_search_readiness_matrix.md @@ -20,7 +20,7 @@ ## Delivery Tracker ### QA-ZL2-001 - Expand deterministic Playwright for the simplified surface -Status: TODO +Status: DONE Dependency: none Owners: Test Automation Task description: @@ -33,7 +33,7 @@ Completion criteria: - [ ] Covered starter chips always land on a meaningful result or explicit grounded fallback. ### QA-ZL2-002 - Run live ingestion-backed suggestion and readiness matrix -Status: TODO +Status: DONE Dependency: QA-ZL2-001 Owners: Test Automation Task description: @@ -46,7 +46,7 @@ Completion criteria: - [ ] Execution logs record the exact rebuild and Playwright commands. ### QA-ZL2-003 - Verify telemetry-off search flows -Status: TODO +Status: DONE Dependency: QA-ZL2-002 Owners: Test Automation Task description: @@ -62,6 +62,7 @@ Completion criteria: | Date (UTC) | Update | Owner | | --- | --- | --- | | 2026-03-07 | Sprint created to keep live-ingested suggestion correctness and telemetry-off behavior as explicit release gates. | Project Manager | +| 2026-03-07 | Added grounded-only live suggestion assertions, telemetry-off deterministic coverage, and reran the combined Playwright gate with live corpus preflight. Evidence: `.artifacts/stella-cli/StellaOps.Cli.exe advisoryai sources prepare --json`; `POST /v1/advisory-ai/index/rebuild`; `POST /v1/search/index/rebuild`; `npx playwright test tests/e2e/unified-search-experience-quality.e2e.spec.ts --config playwright.config.ts`; `LIVE_ADVISORYAI_SEARCH_BASE_URL=http://127.0.0.1:10451 npx playwright test tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts --config playwright.config.ts`; final combined gate `20/20` passed. | Test Automation | ## Decisions & Risks - Decision: live suggestion correctness is a product requirement; deterministic mocks alone are insufficient evidence. diff --git a/src/AdvisoryAI/__Tests/INFRASTRUCTURE.md b/src/AdvisoryAI/__Tests/INFRASTRUCTURE.md index a3d24bf12..d1f07deee 100644 --- a/src/AdvisoryAI/__Tests/INFRASTRUCTURE.md +++ b/src/AdvisoryAI/__Tests/INFRASTRUCTURE.md @@ -148,8 +148,8 @@ Migrations run automatically when the service starts (`EnsureSchemaAsync()`). Or ```bash # Configure connection string for the local AdvisoryAI WebService -export ADVISORYAI__AdvisoryAI__KnowledgeSearch__ConnectionString="Host=localhost;Port=55432;Database=advisoryai_knowledge_test;Username=stellaops_knowledge;Password=stellaops_knowledge" -export ADVISORYAI__AdvisoryAI__KnowledgeSearch__RepositoryRoot="$(pwd)" +export AdvisoryAI__KnowledgeSearch__ConnectionString="Host=localhost;Port=55432;Database=advisoryai_knowledge_test;Username=stellaops_knowledge;Password=stellaops_knowledge" +export AdvisoryAI__KnowledgeSearch__RepositoryRoot="$(pwd)" ``` #### CLI availability in a source checkout diff --git a/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts b/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts index 9cfa17df5..51b47295c 100644 --- a/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts +++ b/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts @@ -141,7 +141,7 @@ test.describe('Unified Search - Live contextual suggestions', () => { }).first()).toBeVisible(); }); - test('every surfaced doctor suggestion executes into a grounded or clarify state', async ({ page }) => { + test('every surfaced doctor suggestion executes into a grounded state', async ({ page }) => { await routeLiveUnifiedSearch(page); await openDoctor(page); @@ -173,7 +173,7 @@ test.describe('Unified Search - Live contextual suggestions', () => { await expect(searchInput).toHaveValue(suggestionText); await waitForResults(page); - await assertNonDeadEndSearch(page, suggestionText); + await assertGroundedSearch(page, suggestionText); } }); @@ -522,12 +522,14 @@ async function buildCompatibilitySuggestionViability( suggestions.push({ query, - viable: cardCount > 0 || status === 'clarify', + viable: status === 'grounded' && cardCount > 0, status, code: String(contextAnswer?.['code'] ?? 'no_grounded_evidence'), cardCount, leadingDomain: leadingDomain || undefined, reason: String(contextAnswer?.['reason'] ?? 'No grounded evidence matched the suggestion in the active corpus.'), + viabilityState: status === 'grounded' ? 'grounded' : status === 'clarify' ? 'needs_clarification' : 'no_match', + scopeReady: cardCount > 0, }); mergedCoverage = mergeCoverage(mergedCoverage, coverage); @@ -584,22 +586,19 @@ function mergeCoverage( }; } -async function assertNonDeadEndSearch(page: Page, suggestionText: string): Promise { +async function assertGroundedSearch(page: Page, suggestionText: string): Promise { await expect.poll(async () => { const status = await page.locator('[data-answer-status]').first().getAttribute('data-answer-status'); - if (status === 'grounded' || status === 'clarify') { + if (status === 'grounded') { return status; } return ''; }, { - message: `Expected "${suggestionText}" to resolve into a grounded or clarify answer.`, + message: `Expected "${suggestionText}" to resolve into a grounded answer.`, }).not.toBe(''); - const answerStatus = await page.locator('[data-answer-status]').first().getAttribute('data-answer-status'); - if (answerStatus === 'grounded') { - await waitForEntityCards(page, 1); - } + await waitForEntityCards(page, 1); } async function mockChatConversation( diff --git a/src/Web/StellaOps.Web/tests/e2e/unified-search-experience-quality.e2e.spec.ts b/src/Web/StellaOps.Web/tests/e2e/unified-search-experience-quality.e2e.spec.ts index d55cea677..bcb5a711a 100644 --- a/src/Web/StellaOps.Web/tests/e2e/unified-search-experience-quality.e2e.spec.ts +++ b/src/Web/StellaOps.Web/tests/e2e/unified-search-experience-quality.e2e.spec.ts @@ -369,6 +369,77 @@ test.describe('Unified Search - Experience Quality UX', () => { expect(lastAction?.['source']).toBe('advisory_ai_chat'); expect(lastAction?.['domain']).toBe('policy'); }); + + test('keeps search, history, and AdvisoryAI handoff working when analytics endpoints are unavailable', async ({ page }) => { + let analyticsAttempts = 0; + + await page.route('**/api/v1/advisory-ai/search/analytics', async (route) => { + analyticsAttempts += 1; + return route.fulfill({ + status: 503, + contentType: 'application/json', + body: JSON.stringify({ error: 'telemetry-disabled' }), + }); + }); + await page.route('**/api/v1/advisory-ai/search/feedback', async (route) => + route.fulfill({ + status: 503, + contentType: 'application/json', + body: JSON.stringify({ error: 'telemetry-disabled' }), + }), + ); + await page.route('**/api/v1/advisory-ai/search/history', async (route) => { + if (route.request().method() === 'DELETE') { + return route.fulfill({ status: 204, body: '' }); + } + + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + entries: [ + { + historyId: 'history-telemetry-off', + query: 'critical findings', + resultCount: 1, + createdAt: '2026-03-07T11:05:00Z', + }, + ], + }), + }); + }); + + await mockSearchResponses(page, (query) => + query.includes('critical findings') + ? criticalFindingResponse + : emptyResponse(query)); + await mockChatConversation(page, { + content: 'Analytics can fail without blocking the search or assistant flow.', + citations: [{ type: 'finding', path: 'CVE-2024-21626', verified: true }], + groundingScore: 0.91, + }); + + await page.goto('/security/triage'); + await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 }); + + const searchInput = page.locator('app-global-search input[type="text"]'); + await searchInput.focus(); + await waitForResults(page); + await expect(page.locator('.search__group').filter({ hasText: 'Recent' })).toContainText('critical findings'); + + await typeInSearch(page, 'critical findings'); + await waitForResults(page); + await waitForEntityCards(page, 1); + await expect(page.locator('[data-answer-status="grounded"]')).toBeVisible(); + await expect.poll(() => analyticsAttempts).toBeGreaterThan(0); + + await page.locator('.search__chat-launcher').click(); + await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 }); + + await searchInput.focus(); + await waitForResults(page); + await expect(page.locator('.search__group').filter({ hasText: 'Recent' })).toContainText('critical findings'); + }); }); async function mockSearchResponses(