diff --git a/docs-archived/implplan/SPRINT_20260307_038_FE_live_search_supported_route_execution_matrix.md b/docs-archived/implplan/SPRINT_20260307_038_FE_live_search_supported_route_execution_matrix.md index a9c5f3e43..08c220caf 100644 --- a/docs-archived/implplan/SPRINT_20260307_038_FE_live_search_supported_route_execution_matrix.md +++ b/docs-archived/implplan/SPRINT_20260307_038_FE_live_search_supported_route_execution_matrix.md @@ -21,7 +21,7 @@ ## Delivery Tracker ### QA-SF-001 - Add supported-route live preflight and corpus readiness checks -Status: TODO +Status: DONE Dependency: none Owners: Test Automation Task description: @@ -29,12 +29,12 @@ Task description: - Fail fast on empty or unsupported corpora instead of letting dead suggestions surface as flaky UI failures. Completion criteria: -- [ ] Live tests verify the rebuild order and supported-route readiness before UI assertions. -- [ ] Unsupported routes are skipped explicitly, not treated as passing suggestion coverage. -- [ ] Empty corpora fail the suite clearly. +- [x] Live tests verify the rebuild order and supported-route readiness before UI assertions. +- [x] Unsupported routes are skipped explicitly, not treated as passing suggestion coverage. +- [x] Empty corpora fail the suite clearly. ### QA-SF-002 - Execute surfaced suggestions on supported routes -Status: TODO +Status: DONE Dependency: QA-SF-001 Owners: Test Automation Task description: @@ -42,12 +42,12 @@ Task description: - Cover assistant handoff from the grounded answer path as part of the same journey. Completion criteria: -- [ ] Every surfaced starter on covered live routes is executed in Playwright. -- [ ] The route-level suite verifies grounded answers or visible useful cards, not just non-empty DOM. -- [ ] Assistant handoff keeps the current page context. +- [x] Every surfaced starter on covered live routes is executed in Playwright. +- [x] The route-level suite verifies grounded answers or visible useful cards, not just non-empty DOM. +- [x] Assistant handoff keeps the current page context. ### QA-SF-003 - Keep deterministic shell regression aligned with live proof -Status: TODO +Status: DONE Dependency: QA-SF-002 Owners: Developer, Test Automation Task description: @@ -55,18 +55,20 @@ Task description: - Keep the suites readable and route-specific rather than one monolithic soak. Completion criteria: -- [ ] Deterministic and live suites assert the same search-first product rules. -- [ ] Covered routes include at least Doctor plus every additional supported ingested route. -- [ ] The execution log records exact commands and results. +- [x] Deterministic and live suites assert the same search-first product rules. +- [x] Covered routes include at least Doctor plus every additional supported ingested route. +- [x] The execution log records exact commands and results. ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | | 2026-03-07 | Sprint created for the supported-route live suggestion execution matrix in the final search-first correction pass. | Project Manager | +| 2026-03-08 | Completed the supported-route live matrix against a rebuilt local corpus and running Angular shell. Evidence: `.artifacts/stella-cli/StellaOps.Cli.exe advisoryai sources prepare --json`; `Invoke-RestMethod -Method Post http://127.0.0.1:10451/v1/advisory-ai/index/rebuild` with `X-StellaOps-Scopes: advisory-ai:admin`, `X-StellaOps-Tenant: test-tenant`, `X-StellaOps-Actor: playwright-live`; `Invoke-RestMethod -Method Post http://127.0.0.1:10451/v1/search/index/rebuild` with the same headers; `npm run serve:test`; `LIVE_ADVISORYAI_SEARCH_BASE_URL=http://127.0.0.1:10451 npx playwright test tests/e2e/unified-search-contextual-suggestions.e2e.spec.ts tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts tests/e2e/unified-search-self-serve-answer-panel.e2e.spec.ts tests/e2e/unified-search-experience-quality.e2e.spec.ts --config playwright.config.ts` -> `24 passed`, `3 skipped`. | Test Automation | ## Decisions & Risks - Decision: suggestion coverage is invalid unless the live corpus is rebuilt and non-empty. - Decision: supported-route proof is part of search release quality, not an optional smoke test. +- Decision: local HTTP rebuilds require explicit StellaOps tenant/scope headers; `sources prepare` is the local CLI step, but index rebuilds remain authenticated backend operations. - Risk: some routes may not yet have enough ingested corpus support to sustain live suggestion coverage. - Mitigation: preflight route support explicitly and suppress unsupported suggestions in product code. - Reference: `docs/modules/ui/search-zero-learning-primary-entry.md` diff --git a/docs/modules/ui/TASKS.md b/docs/modules/ui/TASKS.md index bc0bff4d8..05c311975 100644 --- a/docs/modules/ui/TASKS.md +++ b/docs/modules/ui/TASKS.md @@ -6,7 +6,6 @@ - `docs/implplan/SPRINT_20260307_006_FE_self_serve_rollout_and_gap_closure.md` - `docs/implplan/SPRINT_20260307_009_DOCS_ui_component_preservation_map.md` - `docs/implplan/SPRINT_20260307_023_DOCS_ui_restoration_topic_shapes.md` -- `docs/implplan/SPRINT_20260307_038_FE_live_search_supported_route_execution_matrix.md` ## Delivery Tasks - [DONE] 041-T1 Root IA/nav rewrite (Mission Control + Ops + Setup) @@ -49,9 +48,9 @@ - [DONE] FE-SF-002 Automatic answer/overflow presentation cleanup - [DONE] FE-SF-003 Suggestion execution and success-only history hardening - [DONE] FE-SF-004 Search-first shell verification coverage -- [TODO] QA-SF-001 Live route preflight and corpus readiness gate -- [TODO] QA-SF-002 Execute surfaced suggestions on supported routes -- [TODO] QA-SF-003 Align deterministic and live search-first matrices +- [DONE] QA-SF-001 Live route preflight and corpus readiness gate +- [DONE] QA-SF-002 Execute surfaced suggestions on supported routes +- [DONE] QA-SF-003 Align deterministic and live search-first matrices - [DONE] DOCS-UCM-001 UI component preservation map scaffold and inventory - [DONE] DOCS-UCM-002 First-pass preservation judgments for unused and weakly surfaced UI components - [DONE] DOCS-UCM-003 Summary tree for keep / merge / wire / archive decisions diff --git a/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts b/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts index 51b47295c..75022ef99 100644 --- a/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts +++ b/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts @@ -1,4 +1,4 @@ -import { expect, test, type Page } from '@playwright/test'; +import { expect, test, type Page, type Route } from '@playwright/test'; import { policyAuthorSession } from '../../src/app/testing'; import { waitForEntityCards, waitForResults } from './unified-search-fixtures'; @@ -7,7 +7,6 @@ const liveSearchBaseUrl = process.env['LIVE_ADVISORYAI_SEARCH_BASE_URL']?.trim() const liveTenant = process.env['LIVE_ADVISORYAI_TENANT']?.trim() || 'test-tenant'; const liveScopes = process.env['LIVE_ADVISORYAI_SCOPES']?.trim() || 'advisory-ai:view advisory-ai:operate advisory-ai:admin'; -const liveSuggestionSeedQueries = ['database connectivity', 'OIDC readiness']; const mockConfig = { authority: { @@ -106,6 +105,56 @@ const mockChecks = { total: 3, }; +type LiveRouteConfig = { + key: string; + label: string; + path: string; + heading: RegExp; + seedQueries: readonly string[]; +}; + +type LiveRouteState = { + config: LiveRouteConfig; + supported: boolean; + scopeReady: boolean; + viableCount: number; + viabilityStates: string[]; + payload: Record; +}; + +const liveRouteConfigs: readonly LiveRouteConfig[] = [ + { + key: 'doctor', + label: 'Doctor', + path: '/ops/operations/doctor', + heading: /doctor diagnostics/i, + seedQueries: ['database connectivity', 'OIDC readiness'], + }, + { + key: 'triage', + label: 'Security triage', + path: '/security/triage', + heading: /security\s*\/\s*triage/i, + seedQueries: ['critical findings', 'reachable vulnerabilities', 'unresolved CVEs'], + }, + { + key: 'policy', + label: 'Policy governance', + path: '/ops/policy', + heading: /policy/i, + seedQueries: ['failing policy gates', 'production deny rules', 'policy exceptions'], + }, + { + key: 'vex', + label: 'Advisories and VEX', + path: '/security/advisories-vex', + heading: /advisories\s*&\s*vex/i, + seedQueries: ['Why is this marked not affected?', 'Which components are covered by this VEX?', 'What evidence conflicts with this VEX?'], + }, +] as const; + +const liveRouteStates = new Map(); + test.describe('Unified Search - Live contextual suggestions', () => { test.describe.configure({ mode: 'serial' }); test.setTimeout(120_000); @@ -115,11 +164,20 @@ test.describe('Unified Search - Live contextual suggestions', () => { testInfo.setTimeout(120_000); await ensureLiveServiceHealthy(liveSearchBaseUrl); await rebuildLiveIndexes(liveSearchBaseUrl); - await assertLiveSuggestionCoverage(liveSearchBaseUrl, liveSuggestionSeedQueries); + liveRouteStates.clear(); + + for (const routeConfig of liveRouteConfigs) { + liveRouteStates.set(routeConfig.key, await evaluateLiveRoute(routeConfig)); + } + + const doctorState = requireLiveRouteState('doctor'); + if (!doctorState.supported) { + throw new Error(`Doctor live suggestion preflight returned no grounded viable suggestions: ${JSON.stringify(doctorState.payload)}`); + } }); test.beforeEach(async ({ page }) => { - await setupDoctorPage(page); + await setupLiveShell(page); }); test('shows only viable live suggestion chips when the doctor page opens', async ({ page }) => { @@ -260,9 +318,88 @@ test.describe('Unified Search - Live contextual suggestions', () => { expect(String(capturedTurnBodies.at(-1)?.['content'] ?? '')).toMatch(/database connectivity/i); expect(String(capturedTurnBodies.at(-1)?.['content'] ?? '')).toMatch(/grounded answer|best next step/i); }); + + for (const routeConfig of liveRouteConfigs.filter((route) => route.key !== 'doctor')) { + test(`${routeConfig.label} suppresses surfaced starter chips when the live route corpus is unready`, async ({ page }) => { + const state = requireLiveRouteState(routeConfig.key); + test.skip(state.supported || state.scopeReady, `${routeConfig.label} live route is no longer corpus-unready.`); + + await routeLiveUnifiedSearch(page); + await openLiveRoute(page, routeConfig); + + const searchInput = page.locator('app-global-search input[type="text"]'); + await searchInput.focus(); + await waitForResults(page); + + await expect(page.locator('.search__context-title')).toContainText(routeConfig.heading); + await expect(page.locator('.search__suggestions .search__chip')).toHaveCount(0); + await expect(page.locator('.search__group-label', { + hasText: /start here/i, + })).toHaveCount(0); + }); + + test(`${routeConfig.label} executes every surfaced suggestion when the live route corpus is ready`, async ({ page }) => { + const state = requireLiveRouteState(routeConfig.key); + test.skip(!state.supported, `${routeConfig.label} live route is not yet suggestion-ready.`); + + await routeLiveUnifiedSearch(page); + await openLiveRoute(page, routeConfig); + + const searchInput = page.locator('app-global-search input[type="text"]'); + await searchInput.focus(); + await waitForResults(page); + + const suggestionChips = page.locator('.search__suggestions .search__chip'); + await expect.poll(async () => await suggestionChips.count(), { + message: `${routeConfig.label} should surface at least one viable starter chip.`, + }).toBeGreaterThan(0); + await expect(suggestionChips.first()).toBeVisible({ timeout: 10_000 }); + + const suggestionTexts = (await suggestionChips.allTextContents()) + .map((text) => text.trim()) + .filter((text) => text.length > 0); + + expect(suggestionTexts.length).toBeGreaterThan(0); + + for (const suggestionText of suggestionTexts) { + await openLiveRoute(page, routeConfig); + await searchInput.focus(); + await waitForResults(page); + + const suggestionChip = page.locator('.search__suggestions .search__chip', { + hasText: new RegExp(`^${escapeRegExp(suggestionText)}$`, 'i'), + }).first(); + + if (await suggestionChip.isVisible().catch(() => false)) { + await suggestionChip.click(); + } else { + await searchInput.fill(suggestionText); + } + + await expect(searchInput).toHaveValue(suggestionText); + await waitForResults(page); + await assertGroundedSearch(page, suggestionText); + } + }); + } }); -async function setupDoctorPage(page: Page): Promise { +async function setupLiveShell(page: Page): Promise { + const jsonStubUnlessDocument = (defaultGetBody: unknown = []): ((route: Route) => Promise) => { + return async (route) => { + if (route.request().resourceType() === 'document') { + await route.fallback(); + return; + } + + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(route.request().method() === 'GET' ? defaultGetBody : {}), + }); + }; + }; + await page.addInitScript((stubSession) => { (window as unknown as { __stellaopsTestSession?: unknown }).__stellaopsTestSession = stubSession; }, doctorSession); @@ -337,15 +474,62 @@ async function setupDoctorPage(page: Page): Promise { }), }), ); + + await page.route('**/api/**', jsonStubUnlessDocument()); + await page.route('**/gateway/**', jsonStubUnlessDocument()); + await page.route('**/policy/**', jsonStubUnlessDocument()); + await page.route('**/scanner/**', jsonStubUnlessDocument()); + await page.route('**/concelier/**', jsonStubUnlessDocument()); + await page.route('**/attestor/**', jsonStubUnlessDocument()); } async function openDoctor(page: Page): Promise { - await page.goto('/ops/operations/doctor', { waitUntil: 'domcontentloaded' }); - await expect(page.getByRole('heading', { name: /doctor diagnostics/i })).toBeVisible({ + await openLiveRoute(page, liveRouteConfigs[0]); +} + +async function openLiveRoute(page: Page, routeConfig: LiveRouteConfig): Promise { + await page.goto(routeConfig.path, { waitUntil: 'domcontentloaded' }); + await expect(page.getByRole('heading', { name: routeConfig.heading }).first()).toBeVisible({ timeout: 15_000, }); } +function requireLiveRouteState(routeKey: string): LiveRouteState { + const state = liveRouteStates.get(routeKey); + if (!state) { + throw new Error(`Missing live route preflight for ${routeKey}.`); + } + + return state; +} + +async function evaluateLiveRoute(routeConfig: LiveRouteConfig): Promise { + const payload = await fetchLiveSuggestionViability(JSON.stringify({ + queries: routeConfig.seedQueries, + ambient: { + currentRoute: routeConfig.path, + }, + })); + + const suggestions = Array.isArray(payload['suggestions']) + ? payload['suggestions'] as Array> + : []; + const viableCount = suggestions.filter((suggestion) => suggestion['viable'] === true).length; + const scopeReady = suggestions.some((suggestion) => suggestion['scopeReady'] === true); + const viabilityStates = suggestions + .map((suggestion) => String(suggestion['viabilityState'] ?? 'unknown')) + .filter((state) => state.length > 0); + + return { + config: routeConfig, + supported: viableCount > 0, + scopeReady, + viableCount, + viabilityStates, + payload, + }; +} + async function routeLiveUnifiedSearch( page: Page, capturedRequests?: Array>, @@ -419,26 +603,6 @@ async function rebuildLiveIndexes(baseUrl: string): Promise { } } -async function assertLiveSuggestionCoverage( - baseUrl: string, - queries: readonly string[], -): Promise { - const payload = await fetchLiveSuggestionViability(JSON.stringify({ - queries, - ambient: { - currentRoute: '/ops/operations/doctor', - }, - })); - const suggestions = Array.isArray(payload['suggestions']) - ? payload['suggestions'] as Array> - : []; - const viableSuggestions = suggestions.filter((suggestion) => suggestion['viable'] === true); - - if (viableSuggestions.length === 0) { - throw new Error(`Live suggestion preflight returned no viable queries: ${JSON.stringify(payload)}`); - } -} - function safeParseRequest(rawBody: string): Record { try { const parsed = JSON.parse(rawBody) as Record; @@ -517,19 +681,42 @@ async function buildCompatibilitySuggestionViability( : null; const cardCount = cards.length + overflowCards.length; const status = String(contextAnswer?.['status'] ?? 'insufficient'); + const coverageDomains = Array.isArray(coverage?.['domains']) + ? coverage['domains'] as Array> + : []; + const currentScopeDomain = String(coverage?.['currentScopeDomain'] ?? ''); + const currentScopeCoverage = coverageDomains.find((domain) => + Boolean(domain['isCurrentScope']) + || String(domain['domain'] ?? '') === currentScopeDomain); + const currentScopeHasVisibleResults = + Boolean(currentScopeCoverage?.['hasVisibleResults']) + || Number(currentScopeCoverage?.['visibleCardCount'] ?? 0) > 0; + const currentScopeHasCandidates = Number(currentScopeCoverage?.['candidateCount'] ?? 0) > 0; + const scopeReady = currentScopeDomain.length === 0 + ? cardCount > 0 + : currentScopeHasVisibleResults || currentScopeHasCandidates; + const viabilityState = !scopeReady && currentScopeDomain.length > 0 + ? 'scope_unready' + : status === 'grounded' && cardCount > 0 + ? 'grounded' + : status === 'clarify' + ? 'needs_clarification' + : 'no_match'; const leadingDomain = String(cards[0]?.['domain'] ?? overflowCards[0]?.['domain'] ?? coverage?.['currentScopeDomain'] ?? ''); suggestions.push({ query, - viable: status === 'grounded' && cardCount > 0, + viable: viabilityState === 'grounded', status, code: String(contextAnswer?.['code'] ?? 'no_grounded_evidence'), cardCount, leadingDomain: leadingDomain || undefined, - reason: String(contextAnswer?.['reason'] ?? 'No grounded evidence matched the suggestion in the active corpus.'), - viabilityState: status === 'grounded' ? 'grounded' : status === 'clarify' ? 'needs_clarification' : 'no_match', - scopeReady: cardCount > 0, + reason: viabilityState === 'scope_unready' + ? `The active route maps to ${currentScopeDomain || 'the current scope'}, but that scope has no ingested search corpus yet.` + : String(contextAnswer?.['reason'] ?? 'No grounded evidence matched the suggestion in the active corpus.'), + viabilityState, + scopeReady, }); mergedCoverage = mergeCoverage(mergedCoverage, coverage);