581 lines
20 KiB
TypeScript
581 lines
20 KiB
TypeScript
import { expect, test, type Page } from '@playwright/test';
|
|
|
|
import {
|
|
buildResponse,
|
|
emptyResponse,
|
|
findingCard,
|
|
policyCard,
|
|
setupAuthenticatedSession,
|
|
setupBasicMocks,
|
|
typeInSearch,
|
|
waitForEntityCards,
|
|
waitForResults,
|
|
} from './unified-search-fixtures';
|
|
|
|
const criticalFindingResponse = buildResponse(
|
|
'critical findings',
|
|
[
|
|
findingCard({
|
|
cveId: 'CVE-2024-21626',
|
|
title: 'CVE-2024-21626 in api-gateway',
|
|
snippet: 'Reachable critical vulnerability detected in production workload.',
|
|
severity: 'critical',
|
|
}),
|
|
],
|
|
{
|
|
summary: 'One critical finding matched. Ask AdvisoryAI for triage guidance.',
|
|
template: 'finding_overview',
|
|
confidence: 'high',
|
|
sourceCount: 1,
|
|
domainsCovered: ['findings'],
|
|
},
|
|
);
|
|
|
|
const correctionResponse = {
|
|
...emptyResponse('critcal findings'),
|
|
suggestions: [{ text: 'critical findings', reason: 'Close match in the active corpus.' }],
|
|
};
|
|
|
|
const policyBlockerResponse = buildResponse(
|
|
'policy blockers for CVE-2024-21626',
|
|
[
|
|
policyCard({
|
|
ruleId: 'POL-118',
|
|
title: 'POL-118 release blocker',
|
|
snippet: 'Production rollout is blocked while this CVE remains unresolved.',
|
|
}),
|
|
],
|
|
{
|
|
summary: 'Policy blockers were found for this CVE.',
|
|
template: 'policy_overview',
|
|
confidence: 'high',
|
|
sourceCount: 1,
|
|
domainsCovered: ['policy'],
|
|
},
|
|
);
|
|
|
|
const weightedOverflowResponse = buildResponse(
|
|
'critical findings',
|
|
[
|
|
findingCard({
|
|
cveId: 'CVE-2024-21626',
|
|
title: 'CVE-2024-21626 in api-gateway',
|
|
snippet: 'Reachable critical vulnerability remains the strongest match for this page.',
|
|
severity: 'critical',
|
|
}),
|
|
],
|
|
undefined,
|
|
{
|
|
contextAnswer: {
|
|
status: 'grounded',
|
|
code: 'retrieved_scope_weighted_evidence',
|
|
summary: 'Current-page findings matched first, with one policy blocker held as related overflow.',
|
|
reason: 'The search weighted the active findings page first.',
|
|
evidence: 'Grounded in 2 sources across Findings and Policy.',
|
|
citations: [
|
|
{
|
|
entityKey: 'cve:CVE-2024-21626',
|
|
title: 'CVE-2024-21626 in api-gateway',
|
|
domain: 'findings',
|
|
},
|
|
],
|
|
questions: [
|
|
{
|
|
query: 'What evidence blocks this release?',
|
|
kind: 'follow_up',
|
|
},
|
|
],
|
|
},
|
|
overflow: {
|
|
currentScopeDomain: 'findings',
|
|
reason: 'Policy results remain relevant but are weaker than the current findings context.',
|
|
cards: [
|
|
policyCard({
|
|
ruleId: 'POL-118',
|
|
title: 'POL-118 release blocker',
|
|
snippet: 'Production rollout is blocked while this CVE remains unresolved.',
|
|
}),
|
|
],
|
|
},
|
|
coverage: {
|
|
currentScopeDomain: 'findings',
|
|
currentScopeWeighted: true,
|
|
domains: [
|
|
{
|
|
domain: 'findings',
|
|
candidateCount: 3,
|
|
visibleCardCount: 1,
|
|
topScore: 0.96,
|
|
isCurrentScope: true,
|
|
hasVisibleResults: true,
|
|
},
|
|
{
|
|
domain: 'policy',
|
|
candidateCount: 1,
|
|
visibleCardCount: 1,
|
|
topScore: 0.74,
|
|
isCurrentScope: false,
|
|
hasVisibleResults: true,
|
|
},
|
|
],
|
|
},
|
|
},
|
|
);
|
|
|
|
test.describe('Unified Search - Experience Quality UX', () => {
|
|
test.beforeEach(async ({ page }) => {
|
|
await setupBasicMocks(page);
|
|
await setupAuthenticatedSession(page);
|
|
});
|
|
|
|
test('opens AdvisoryAI from the secondary search-bar launcher with page and query context', async ({ page }) => {
|
|
const capturedTurnBodies: Array<Record<string, unknown>> = [];
|
|
await mockSearchResponses(page, (query) => {
|
|
if (query.includes('critical findings')) {
|
|
return criticalFindingResponse;
|
|
}
|
|
|
|
return emptyResponse(query);
|
|
});
|
|
await mockChatConversation(page, {
|
|
content: 'AdvisoryAI is ready to expand the answer and explain the next step.',
|
|
citations: [{ type: 'finding', path: 'CVE-2024-21626', verified: true }],
|
|
groundingScore: 0.94,
|
|
onTurnCreate: (body) => capturedTurnBodies.push(body),
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await typeInSearch(page, 'critical findings');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
|
|
await page.locator('.search__chat-launcher').click();
|
|
|
|
await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 });
|
|
await expect(page.locator('.header-title')).toContainText('Search assistant');
|
|
await expect(page.locator('app-global-search input[type="text"]')).toHaveValue('');
|
|
await expect.poll(() => capturedTurnBodies.length).toBeGreaterThan(0);
|
|
|
|
const turnBody = capturedTurnBodies.at(-1) ?? {};
|
|
expect(String(turnBody['content'] ?? '')).toMatch(/critical findings/i);
|
|
});
|
|
|
|
test('keeps the empty state to page context, recent history, and executable starters only', async ({ page }) => {
|
|
await mockSearchResponses(page, (query) => {
|
|
if (query.includes('critical findings')) {
|
|
return criticalFindingResponse;
|
|
}
|
|
|
|
return emptyResponse(query);
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
const searchInput = page.locator('app-global-search input[type="text"]');
|
|
await searchInput.focus();
|
|
await waitForResults(page);
|
|
|
|
await expect(page.locator('.search__context-title')).toContainText(/findings triage/i);
|
|
await expect(page.locator('.search__domain-guide')).toHaveCount(0);
|
|
await expect(page.locator('.search__quick-link')).toHaveCount(0);
|
|
|
|
const starterChip = page.locator('[data-starter-kind]', {
|
|
hasText: /critical findings/i,
|
|
}).first();
|
|
await expect(starterChip).toBeVisible();
|
|
|
|
await starterChip.click();
|
|
await expect(searchInput).toHaveValue(/critical findings/i);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
});
|
|
|
|
test('suppresses a starter chip after it executes to no results and keeps it out of history', async ({ page }) => {
|
|
await page.route('**/search/query**', async (route) => {
|
|
const request = route.request().postDataJSON() as Record<string, unknown>;
|
|
const query = String(request['q'] ?? '').toLowerCase();
|
|
const response = query.includes('critical findings')
|
|
? emptyResponse('critical findings')
|
|
: criticalFindingResponse;
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(response),
|
|
});
|
|
});
|
|
|
|
await page.route('**/api/v1/advisory-ai/search/history', async (route) => {
|
|
if (route.request().method() === 'DELETE') {
|
|
return route.fulfill({ status: 204, body: '' });
|
|
}
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ entries: [] }),
|
|
});
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
const searchInput = page.locator('app-global-search input[type="text"]');
|
|
await searchInput.focus();
|
|
await waitForResults(page);
|
|
|
|
const failingChip = page.locator('[data-starter-kind]', { hasText: /critical findings/i }).first();
|
|
await expect(failingChip).toBeVisible();
|
|
|
|
await failingChip.click();
|
|
await waitForResults(page);
|
|
await expect(page.locator('.search__empty')).toContainText('No results found');
|
|
|
|
await searchInput.fill('');
|
|
await waitForResults(page);
|
|
|
|
await expect(page.locator('[data-starter-kind]', { hasText: /critical findings/i })).toHaveCount(0);
|
|
await expect(page.locator('.search__group').filter({ hasText: 'Recent' })).toHaveCount(0);
|
|
});
|
|
|
|
test('renders did-you-mean directly below the search bar and removes teaching controls', async ({ page }) => {
|
|
const capturedRequests: Array<Record<string, unknown>> = [];
|
|
await page.route('**/search/query**', async (route) => {
|
|
const request = route.request().postDataJSON() as Record<string, unknown>;
|
|
capturedRequests.push(request);
|
|
const query = String(request['q'] ?? '').toLowerCase();
|
|
const response = query.includes('critcal findings')
|
|
? correctionResponse
|
|
: criticalFindingResponse;
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(response),
|
|
});
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await typeInSearch(page, 'critcal findings');
|
|
await waitForResults(page);
|
|
|
|
const searchBar = page.locator('.search__input-wrapper');
|
|
const didYouMean = page.locator('.did-you-mean--inline');
|
|
|
|
await expect(didYouMean).toBeVisible();
|
|
await expect(page.locator('.search__segment')).toHaveCount(0);
|
|
await expect(page.locator('.search__scope-chip')).toHaveCount(0);
|
|
await expect(page.locator('.search__rescue-card')).toHaveCount(0);
|
|
await expect(page.locator('.search__domain-guide')).toHaveCount(0);
|
|
await expect(page.locator('.search__quick-link')).toHaveCount(0);
|
|
|
|
const searchBarBox = await searchBar.boundingBox();
|
|
const didYouMeanBox = await didYouMean.boundingBox();
|
|
expect(searchBarBox).not.toBeNull();
|
|
expect(didYouMeanBox).not.toBeNull();
|
|
expect(didYouMeanBox!.y).toBeGreaterThan(searchBarBox!.y);
|
|
expect(didYouMeanBox!.y - (searchBarBox!.y + searchBarBox!.height)).toBeLessThan(20);
|
|
|
|
const request = capturedRequests[0] ?? {};
|
|
expect(request['filters']).toBeUndefined();
|
|
});
|
|
|
|
test('shows only successful history entries and clears them with the icon action', async ({ page }) => {
|
|
let historyCleared = false;
|
|
await mockSearchResponses(page, (query) => {
|
|
if (query.includes('critical findings')) {
|
|
return criticalFindingResponse;
|
|
}
|
|
|
|
return emptyResponse(query);
|
|
});
|
|
await page.route('**/api/v1/advisory-ai/search/history', async (route) => {
|
|
if (route.request().method() === 'DELETE') {
|
|
historyCleared = true;
|
|
return route.fulfill({ status: 204, body: '' });
|
|
}
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
entries: [
|
|
{
|
|
historyId: 'history-1',
|
|
query: 'critical findings',
|
|
resultCount: 2,
|
|
createdAt: '2026-03-07T11:00:00Z',
|
|
},
|
|
{
|
|
historyId: 'history-2',
|
|
query: 'database connectivity',
|
|
resultCount: 0,
|
|
createdAt: '2026-03-07T11:01:00Z',
|
|
},
|
|
],
|
|
}),
|
|
});
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
|
|
await expect(page.locator('.search__group').filter({ hasText: 'Recent' })).toContainText('critical findings');
|
|
await expect(page.locator('.search__group').filter({ hasText: 'Recent' })).not.toContainText('database connectivity');
|
|
|
|
const clearButton = page.locator('.search__clear-history');
|
|
await expect(clearButton).toBeVisible();
|
|
await expect(clearButton.locator('svg')).toBeVisible();
|
|
await expect(clearButton).toHaveText('');
|
|
|
|
await clearButton.click();
|
|
|
|
await expect.poll(() => historyCleared).toBe(true);
|
|
await expect(page.locator('.search__group').filter({ hasText: 'Recent' })).toHaveCount(0);
|
|
});
|
|
|
|
test('uses backend answer framing and shows overflow as secondary results without manual filters', async ({ page }) => {
|
|
await mockSearchResponses(page, (query) =>
|
|
query.includes('critical findings')
|
|
? weightedOverflowResponse
|
|
: emptyResponse(query));
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await typeInSearch(page, 'critical findings');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
|
|
await expect(page.locator('[data-answer-status="grounded"]')).toContainText(
|
|
/current-page findings matched first/i,
|
|
);
|
|
await expect(page.locator('.search__scope-hint')).toHaveCount(0);
|
|
await expect(page.locator('[data-overflow-results]')).toContainText(/also relevant elsewhere/i);
|
|
await expect(page.locator('[data-overflow-results]')).toContainText(/policy results remain relevant/i);
|
|
await expect(page.locator('[data-role="domain-filter"]')).toHaveCount(0);
|
|
await expect(page.locator('app-synthesis-panel')).toHaveCount(0);
|
|
});
|
|
|
|
test('returns structured next-step policy searches back into global search with metadata', async ({ page }) => {
|
|
const capturedRequests: Array<Record<string, unknown>> = [];
|
|
await page.route('**/search/query**', async (route) => {
|
|
const request = route.request().postDataJSON() as Record<string, unknown>;
|
|
capturedRequests.push(request);
|
|
|
|
const query = String(request['q'] ?? '').toLowerCase();
|
|
const response = query.includes('policy blockers for cve-2024-21626')
|
|
? policyBlockerResponse
|
|
: criticalFindingResponse;
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(response),
|
|
});
|
|
});
|
|
await mockChatConversation(page, {
|
|
content: 'CVE-2024-21626 is still gating release decisions and policy evidence should be checked.',
|
|
citations: [
|
|
{ type: 'finding', path: 'CVE-2024-21626', verified: true },
|
|
{ type: 'policy', path: 'POL-118', verified: true },
|
|
],
|
|
groundingScore: 0.96,
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await typeInSearch(page, 'critical findings');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
await page.locator('.entity-card__action--ask-ai').first().click();
|
|
|
|
await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 });
|
|
await expect(page.locator('[data-next-step="policy"]')).toBeVisible({ timeout: 10_000 });
|
|
|
|
await page.locator('[data-next-step="policy"]').click();
|
|
|
|
await expect(page.locator('.assistant-drawer')).toBeHidden({ timeout: 10_000 });
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
await expect(page.locator('app-global-search input[type="text"]')).toHaveValue(/policy .*CVE-2024-21626/i);
|
|
|
|
const policyRequest = capturedRequests.find((request) =>
|
|
/policy .*cve-2024-21626/i.test(String(request['q'] ?? '')));
|
|
const ambient = policyRequest?.['ambient'] as Record<string, unknown> | undefined;
|
|
const lastAction = ambient?.['lastAction'] as Record<string, unknown> | undefined;
|
|
|
|
expect(lastAction?.['action']).toBe('chat_next_step_policy');
|
|
expect(lastAction?.['source']).toBe('advisory_ai_chat');
|
|
expect(lastAction?.['domain']).toBe('policy');
|
|
});
|
|
|
|
test('keeps search, history, and AdvisoryAI handoff working when analytics endpoints are unavailable', async ({ page }) => {
|
|
let analyticsAttempts = 0;
|
|
|
|
await page.route('**/api/v1/advisory-ai/search/analytics', async (route) => {
|
|
analyticsAttempts += 1;
|
|
return route.fulfill({
|
|
status: 503,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ error: 'telemetry-disabled' }),
|
|
});
|
|
});
|
|
await page.route('**/api/v1/advisory-ai/search/feedback', async (route) =>
|
|
route.fulfill({
|
|
status: 503,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({ error: 'telemetry-disabled' }),
|
|
}),
|
|
);
|
|
await page.route('**/api/v1/advisory-ai/search/history', async (route) => {
|
|
if (route.request().method() === 'DELETE') {
|
|
return route.fulfill({ status: 204, body: '' });
|
|
}
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
entries: [
|
|
{
|
|
historyId: 'history-telemetry-off',
|
|
query: 'critical findings',
|
|
resultCount: 1,
|
|
createdAt: '2026-03-07T11:05:00Z',
|
|
},
|
|
],
|
|
}),
|
|
});
|
|
});
|
|
|
|
await mockSearchResponses(page, (query) =>
|
|
query.includes('critical findings')
|
|
? criticalFindingResponse
|
|
: emptyResponse(query));
|
|
await mockChatConversation(page, {
|
|
content: 'Analytics can fail without blocking the search or assistant flow.',
|
|
citations: [{ type: 'finding', path: 'CVE-2024-21626', verified: true }],
|
|
groundingScore: 0.91,
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
const searchInput = page.locator('app-global-search input[type="text"]');
|
|
await searchInput.focus();
|
|
await waitForResults(page);
|
|
await expect(page.locator('.search__group').filter({ hasText: 'Recent' })).toContainText('critical findings');
|
|
|
|
await typeInSearch(page, 'critical findings');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
await expect(page.locator('[data-answer-status="grounded"]')).toBeVisible();
|
|
await expect.poll(() => analyticsAttempts).toBeGreaterThan(0);
|
|
|
|
await page.locator('.search__chat-launcher').click();
|
|
await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 });
|
|
|
|
await searchInput.focus();
|
|
await waitForResults(page);
|
|
await expect(page.locator('.search__group').filter({ hasText: 'Recent' })).toContainText('critical findings');
|
|
});
|
|
});
|
|
|
|
async function mockSearchResponses(
|
|
page: Page,
|
|
resolve: (normalizedQuery: string) => unknown,
|
|
): Promise<void> {
|
|
await page.route('**/search/query**', async (route) => {
|
|
const body = route.request().postDataJSON() as Record<string, unknown>;
|
|
const query = String(body['q'] ?? '').toLowerCase();
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(resolve(query)),
|
|
});
|
|
});
|
|
}
|
|
|
|
async function mockChatConversation(
|
|
page: Page,
|
|
response: {
|
|
content: string;
|
|
citations: Array<{ type: string; path: string; verified: boolean }>;
|
|
groundingScore: number;
|
|
onConversationCreate?: (body: Record<string, unknown>) => void;
|
|
onTurnCreate?: (body: Record<string, unknown>) => void;
|
|
},
|
|
): Promise<void> {
|
|
await page.route('**/api/v1/advisory-ai/conversations', async (route) => {
|
|
if (route.request().method() !== 'POST') {
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([]),
|
|
});
|
|
}
|
|
|
|
const requestBody = (route.request().postDataJSON() as Record<string, unknown> | null) ?? {};
|
|
response.onConversationCreate?.(requestBody);
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
conversationId: 'conv-quality-1',
|
|
tenantId: 'test-tenant',
|
|
userId: 'tester',
|
|
context: {},
|
|
turns: [],
|
|
createdAt: '2026-03-06T00:00:00.000Z',
|
|
updatedAt: '2026-03-06T00:00:00.000Z',
|
|
}),
|
|
});
|
|
});
|
|
|
|
await page.route('**/api/v1/advisory-ai/conversations/*/turns', async (route) => {
|
|
if (route.request().method() !== 'POST') {
|
|
return route.continue();
|
|
}
|
|
|
|
const requestBody = (route.request().postDataJSON() as Record<string, unknown> | null) ?? {};
|
|
response.onTurnCreate?.(requestBody);
|
|
|
|
const events = [
|
|
'event: progress',
|
|
'data: {"stage":"searching"}',
|
|
'',
|
|
'event: token',
|
|
`data: ${JSON.stringify({ content: response.content })}`,
|
|
'',
|
|
...response.citations.flatMap((citation) => ([
|
|
'event: citation',
|
|
`data: ${JSON.stringify(citation)}`,
|
|
'',
|
|
])),
|
|
'event: done',
|
|
`data: ${JSON.stringify({ turnId: 'turn-quality-1', groundingScore: response.groundingScore })}`,
|
|
'',
|
|
].join('\n');
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
headers: {
|
|
'content-type': 'text/event-stream; charset=utf-8',
|
|
'cache-control': 'no-cache',
|
|
},
|
|
body: events,
|
|
});
|
|
});
|
|
}
|