diff --git a/src/Web/StellaOps.Web/tests/e2e/unified-search-exhaustive-matrix.e2e.spec.ts b/src/Web/StellaOps.Web/tests/e2e/unified-search-exhaustive-matrix.e2e.spec.ts new file mode 100644 index 000000000..668a9b3dd --- /dev/null +++ b/src/Web/StellaOps.Web/tests/e2e/unified-search-exhaustive-matrix.e2e.spec.ts @@ -0,0 +1,417 @@ +import { expect, test, type Locator, type Page } from '@playwright/test'; +import { + buildResponse, + setupAuthenticatedSession, + setupBasicMocks, +} from './unified-search-fixtures'; + +type MatrixDomain = + | 'findings' + | 'vex' + | 'policy' + | 'knowledge' + | 'graph' + | 'timeline' + | 'ops_memory' + | 'platform'; + +interface MatrixKeyword { + keyword: string; + domain: MatrixDomain; +} + +const intents = ['show', 'list', 'find', 'explain'] as const; + +const qualifiers = [ + 'in production', + 'in staging', + 'in dev', + 'from last 24h', + 'from last 7d', + 'for api-gateway', + 'for payments-service', + 'with critical severity', + 'with policy impact', + 'with remediation steps', +] as const; + +const keywords: readonly MatrixKeyword[] = [ + { keyword: 'CVE findings', domain: 'findings' }, + { keyword: 'reachable vulnerabilities', domain: 'findings' }, + { keyword: 'secret detections', domain: 'findings' }, + { keyword: 'VEX not affected statements', domain: 'vex' }, + { keyword: 'VEX affected assertions', domain: 'vex' }, + { keyword: 'VEX under investigation records', domain: 'vex' }, + { keyword: 'policy deny rules', domain: 'policy' }, + { keyword: 'policy exceptions', domain: 'policy' }, + { keyword: 'policy gate failures', domain: 'policy' }, + { keyword: 'deployment docs', domain: 'knowledge' }, + { keyword: 'api endpoint contracts', domain: 'knowledge' }, + { keyword: 'doctor health checks', domain: 'knowledge' }, + { keyword: 'reachability graph paths', domain: 'graph' }, + { keyword: 'dependency graph nodes', domain: 'graph' }, + { keyword: 'blast radius graph edges', domain: 'graph' }, + { keyword: 'incident timelines', domain: 'timeline' }, + { keyword: 'promotion history events', domain: 'timeline' }, + { keyword: 'audit timeline markers', domain: 'timeline' }, + { keyword: 'job scheduler runs', domain: 'ops_memory' }, + { keyword: 'task runner outcomes', domain: 'ops_memory' }, + { keyword: 'ops memory traces', domain: 'ops_memory' }, + { keyword: 'platform integrations', domain: 'platform' }, + { keyword: 'environment readiness', domain: 'platform' }, + { keyword: 'release orchestration status', domain: 'platform' }, + { keyword: 'runtime policy drift', domain: 'policy' }, + { keyword: 'sbom provenance docs', domain: 'knowledge' }, + { keyword: 'scanner finding feed', domain: 'findings' }, + { keyword: 'vulnerability exploitability records', domain: 'findings' }, + { keyword: 'timeline causality links', domain: 'timeline' }, + { keyword: 'graph risk pivots', domain: 'graph' }, +] as const; + +const matrixQueries = buildQueryMatrix(); +const matrixBatches = chunkQueries(matrixQueries, 200); + +test.describe('Unified Search - Exhaustive Query Matrix', () => { + test('covers a matrix larger than 1000 query types', async () => { + expect(matrixQueries.length).toBeGreaterThan(1000); + expect(matrixBatches.length).toBeGreaterThanOrEqual(6); + }); + + for (const [batchIndex, batch] of matrixBatches.entries()) { + test(`batch ${batchIndex + 1}/${matrixBatches.length} executes ${batch.length} search types with 100% success`, async ({ page }) => { + test.setTimeout(4 * 60 * 1000); + + await setupBasicMocks(page); + await setupAuthenticatedSession(page); + await mockSearchSupportEndpoints(page); + await mockMatrixSearchApi(page); + + await page.goto('/security/triage'); + await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 }); + + const searchInput = page.locator('app-global-search input[type="text"]'); + await searchInput.focus(); + await expect(page.locator('.search__results')).toBeVisible({ timeout: 10_000 }); + + const failures: string[] = []; + + for (const query of batch) { + const normalizedQuery = query.trim(); + const querySubmitted = await submitQueryAndWaitForSearchResponse( + page, + searchInput, + normalizedQuery, + ); + if (!querySubmitted) { + failures.push(`${normalizedQuery} (no search response)`); + continue; + } + + const firstCard = page.locator('app-entity-card').first(); + try { + await firstCard.waitFor({ state: 'visible', timeout: 4_000 }); + } catch { + failures.push(`${normalizedQuery} (no visible result card)`); + } + + await page.waitForTimeout(25); + } + + // eslint-disable-next-line no-console + console.log(`[matrix] batch ${batchIndex + 1}/${matrixBatches.length} processed ${batch.length} queries`); + + expect( + failures, + `Failed queries: ${failures.slice(0, 20).join(' | ')}`, + ).toEqual([]); + }); + } +}); + +function buildQueryMatrix(): string[] { + const generated: string[] = []; + for (const keyword of keywords) { + for (const intent of intents) { + for (const qualifier of qualifiers) { + generated.push(`${intent} ${keyword.keyword} ${qualifier}`); + } + } + } + + return Array.from(new Set(generated)); +} + +function chunkQueries(values: readonly string[], size: number): string[][] { + const output: string[][] = []; + for (let index = 0; index < values.length; index += size) { + output.push(values.slice(index, index + size)); + } + + return output; +} + +async function mockSearchSupportEndpoints(page: Page): Promise { + await page.route('**/api/v1/advisory-ai/search/history**', async (route) => { + const method = route.request().method().toUpperCase(); + if (method === 'GET') { + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ entries: [] }), + }); + } + + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({}), + }); + }); + + await page.route('**/api/v1/advisory-ai/search/analytics**', async (route) => + route.fulfill({ + status: 202, + contentType: 'application/json', + body: JSON.stringify({ accepted: true }), + })); + + await page.route('**/api/v1/advisory-ai/search/feedback**', async (route) => + route.fulfill({ + status: 202, + contentType: 'application/json', + body: JSON.stringify({ accepted: true }), + })); +} + +async function mockMatrixSearchApi(page: Page): Promise { + await page.route('**/search/query**', async (route) => { + const payload = route.request().postDataJSON() as { q?: string }; + const query = String(payload?.q ?? '').trim(); + const domain = classifyDomain(query); + const response = buildResponse( + query, + [buildCard(domain, query)], + { + summary: `${domain} result for "${query}".`, + template: `${domain}_summary`, + confidence: 'high', + sourceCount: 1, + domainsCovered: [domain], + }, + ); + + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(response), + }); + }); +} + +async function submitQueryAndWaitForSearchResponse( + page: Page, + searchInput: Locator, + query: string, +): Promise { + for (let attempt = 0; attempt < 6; attempt++) { + await searchInput.focus(); + + const responsePromise = page.waitForResponse( + (response) => + response.url().includes('/api/v1/search/query') && + response.request().method().toUpperCase() === 'POST', + { timeout: 10_000 }, + ); + + if (attempt === 0) { + await searchInput.fill(query); + } else if (attempt === 1) { + // Retry path: perturb input to force ngModel/debounce pipeline to emit again. + await searchInput.fill(`${query} `); + await searchInput.press('Backspace'); + } else { + // Fallback retry path: clear and re-enter query. + await searchInput.fill(''); + if (attempt === 4) { + await searchInput.type(query, { delay: 8 }); + } else { + await searchInput.fill(query); + } + } + + try { + await responsePromise; + return true; + } catch { + await page.waitForTimeout(80); + } + } + + return false; +} + +function classifyDomain(query: string): MatrixDomain { + const normalized = query.toLowerCase(); + const keyword = keywords.find((entry) => normalized.includes(entry.keyword.toLowerCase())); + return keyword?.domain ?? 'knowledge'; +} + +function buildCard(domain: MatrixDomain, query: string) { + const normalizedKey = query.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, ''); + const base = { + entityKey: `${domain}:${normalizedKey || 'query'}`, + title: `${domain.toUpperCase()} result for ${query}`, + snippet: `Deterministic ${domain} fixture for ${query}.`, + score: 0.9, + isPrimary: true, + sources: [domain], + }; + + switch (domain) { + case 'findings': + return { + entityKey: base.entityKey, + entityType: 'finding', + domain: 'findings', + title: base.title, + snippet: base.snippet, + score: base.score, + severity: 'high', + actions: [ + { + label: 'View Finding', + actionType: 'navigate', + route: `/security/triage?q=${encodeURIComponent(query)}`, + isPrimary: base.isPrimary, + }, + ], + sources: base.sources, + }; + case 'vex': + return { + entityKey: base.entityKey, + entityType: 'vex_statement', + domain: 'vex', + title: base.title, + snippet: base.snippet, + score: base.score, + actions: [ + { + label: 'View VEX', + actionType: 'navigate', + route: `/security/advisories-vex?q=${encodeURIComponent(query)}`, + isPrimary: base.isPrimary, + }, + ], + sources: base.sources, + }; + case 'policy': + return { + entityKey: base.entityKey, + entityType: 'policy_rule', + domain: 'policy', + title: base.title, + snippet: base.snippet, + score: base.score, + actions: [ + { + label: 'View Policy', + actionType: 'navigate', + route: `/ops/policy?q=${encodeURIComponent(query)}`, + isPrimary: base.isPrimary, + }, + ], + sources: base.sources, + }; + case 'graph': + return { + entityKey: base.entityKey, + entityType: 'graph_node', + domain: 'graph', + title: base.title, + snippet: base.snippet, + score: base.score, + actions: [ + { + label: 'Open Graph', + actionType: 'navigate', + route: '/ops/graph', + isPrimary: base.isPrimary, + }, + ], + sources: base.sources, + }; + case 'timeline': + return { + entityKey: base.entityKey, + entityType: 'ops_event', + domain: 'timeline', + title: base.title, + snippet: base.snippet, + score: base.score, + actions: [ + { + label: 'Open Timeline', + actionType: 'navigate', + route: '/ops/timeline', + isPrimary: base.isPrimary, + }, + ], + sources: base.sources, + }; + case 'ops_memory': + return { + entityKey: base.entityKey, + entityType: 'ops_event', + domain: 'ops_memory', + title: base.title, + snippet: base.snippet, + score: base.score, + actions: [ + { + label: 'Open Jobs', + actionType: 'navigate', + route: '/ops/operations/jobs', + isPrimary: base.isPrimary, + }, + ], + sources: base.sources, + }; + case 'platform': + return { + entityKey: base.entityKey, + entityType: 'platform_entity', + domain: 'platform', + title: base.title, + snippet: base.snippet, + score: base.score, + actions: [ + { + label: 'Open Platform', + actionType: 'navigate', + route: '/mission-control', + isPrimary: base.isPrimary, + }, + ], + sources: base.sources, + }; + default: + return { + entityKey: base.entityKey, + entityType: query.toLowerCase().includes('api') ? 'api' : 'docs', + domain: 'knowledge', + title: base.title, + snippet: base.snippet, + score: base.score, + actions: [ + { + label: 'Open Docs', + actionType: 'navigate', + route: '/docs', + isPrimary: base.isPrimary, + }, + ], + sources: base.sources, + }; + } +}