321 lines
11 KiB
TypeScript
321 lines
11 KiB
TypeScript
import { expect, test, type Page } from '@playwright/test';
|
|
|
|
import {
|
|
buildResponse,
|
|
emptyResponse,
|
|
findingCard,
|
|
policyCard,
|
|
setupAuthenticatedSession,
|
|
setupBasicMocks,
|
|
typeInSearch,
|
|
waitForEntityCards,
|
|
waitForResults,
|
|
} from './unified-search-fixtures';
|
|
|
|
const criticalFindingResponse = buildResponse(
|
|
'critical findings',
|
|
[
|
|
findingCard({
|
|
cveId: 'CVE-2024-21626',
|
|
title: 'CVE-2024-21626 in api-gateway',
|
|
snippet: 'Reachable critical vulnerability detected in production workload.',
|
|
severity: 'critical',
|
|
}),
|
|
],
|
|
{
|
|
summary: 'One critical finding matched. Ask AdvisoryAI for triage guidance.',
|
|
template: 'finding_overview',
|
|
confidence: 'high',
|
|
sourceCount: 1,
|
|
domainsCovered: ['findings'],
|
|
},
|
|
);
|
|
|
|
const broadenedScopeResponse = buildResponse(
|
|
'scope sensitive outage',
|
|
[
|
|
policyCard({
|
|
ruleId: 'DENY-CRITICAL-PROD',
|
|
title: 'DENY-CRITICAL-PROD',
|
|
snippet: 'Production deny rule linked to the active incident.',
|
|
}),
|
|
],
|
|
{
|
|
summary: 'The broader search found a policy blocker outside the page scope.',
|
|
template: 'policy_overview',
|
|
confidence: 'high',
|
|
sourceCount: 1,
|
|
domainsCovered: ['policy'],
|
|
},
|
|
);
|
|
|
|
const policyBlockerResponse = buildResponse(
|
|
'policy blockers for CVE-2024-21626',
|
|
[
|
|
policyCard({
|
|
ruleId: 'POL-118',
|
|
title: 'POL-118 release blocker',
|
|
snippet: 'Production rollout is blocked while this CVE remains unresolved.',
|
|
}),
|
|
],
|
|
{
|
|
summary: 'Policy blockers were found for this CVE.',
|
|
template: 'policy_overview',
|
|
confidence: 'high',
|
|
sourceCount: 1,
|
|
domainsCovered: ['policy'],
|
|
},
|
|
);
|
|
|
|
test.describe('Unified Search - Experience Quality UX', () => {
|
|
test.beforeEach(async ({ page }) => {
|
|
await setupBasicMocks(page);
|
|
await setupAuthenticatedSession(page);
|
|
});
|
|
|
|
test('keeps keyboard-selected mode when handing off from search to AdvisoryAI', async ({ page }) => {
|
|
await mockSearchResponses(page, (query) => {
|
|
if (query.includes('critical findings')) {
|
|
return criticalFindingResponse;
|
|
}
|
|
|
|
return emptyResponse(query);
|
|
});
|
|
await mockChatConversation(page, {
|
|
content: 'AdvisoryAI is ready to explain the finding and cite evidence.',
|
|
citations: [{ type: 'finding', path: 'CVE-2024-21626', verified: true }],
|
|
groundingScore: 0.94,
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await typeInSearch(page, 'critical findings');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
|
|
const explainButton = page.locator('.search__experience-bar').getByRole('button', { name: 'Explain' });
|
|
await explainButton.focus();
|
|
await explainButton.press('Enter');
|
|
await expect(explainButton).toHaveClass(/search__segment--active/);
|
|
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
await page.locator('.entity-card__action--ask-ai').first().click();
|
|
|
|
await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 });
|
|
await expect(page.locator('.chat-mode-btn--active')).toHaveText(/Explain/i);
|
|
});
|
|
|
|
test('broadens zero-result searches to all domains and reruns the same query', async ({ page }) => {
|
|
const capturedRequests: Array<Record<string, unknown>> = [];
|
|
await page.route('**/search/query**', async (route) => {
|
|
const request = route.request().postDataJSON() as Record<string, unknown>;
|
|
capturedRequests.push(request);
|
|
|
|
const query = String(request['q'] ?? '').toLowerCase();
|
|
const filters = request['filters'] as Record<string, unknown> | undefined;
|
|
const hasPageScope = Array.isArray(filters?.['domains']) && filters!['domains'].length > 0;
|
|
|
|
const response = query.includes('scope sensitive outage')
|
|
? hasPageScope
|
|
? emptyResponse('scope sensitive outage')
|
|
: broadenedScopeResponse
|
|
: emptyResponse(query);
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(response),
|
|
});
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await typeInSearch(page, 'scope sensitive outage');
|
|
await waitForResults(page);
|
|
await expect(page.locator('.search__empty')).toContainText(/no results found/i);
|
|
await expect(page.locator('.search__rescue-card')).toHaveCount(4);
|
|
|
|
await page.locator('[data-rescue-action="scope"]').click();
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
await expect(page.locator('[data-role="search-scope"]')).toContainText(/All domains/i);
|
|
await expect(page.locator('app-global-search input[type="text"]')).toHaveValue('scope sensitive outage');
|
|
await expect(page.locator('.search__cards')).toContainText(/DENY-CRITICAL-PROD/i);
|
|
|
|
expect(capturedRequests[0]?.['q']).toBe('scope sensitive outage');
|
|
});
|
|
|
|
test('opens AdvisoryAI reformulation from the zero-result rescue flow', async ({ page }) => {
|
|
await mockSearchResponses(page, (query) =>
|
|
query.includes('mystery remediation') ? emptyResponse('mystery remediation') : emptyResponse(query));
|
|
await mockChatConversation(page, {
|
|
content: 'I can reformulate that query for better recall.',
|
|
citations: [{ type: 'docs', path: 'modules/ui/search-chip-context-contract.md', verified: true }],
|
|
groundingScore: 0.91,
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
await page.locator('.search__experience-bar').getByRole('button', { name: 'Explain' }).click();
|
|
await typeInSearch(page, 'mystery remediation');
|
|
await waitForResults(page);
|
|
await expect(page.locator('.search__empty')).toContainText(/no results found/i);
|
|
|
|
await page.locator('[data-rescue-action="reformulate"]').click();
|
|
|
|
await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 });
|
|
await expect(page.locator('.chat-mode-btn--active')).toHaveText(/Explain/i);
|
|
await expect(page.locator('.chat-message.user .message-body').first()).toContainText(
|
|
/Reformulate the search query "mystery remediation"/i,
|
|
);
|
|
});
|
|
|
|
test('returns structured next-step policy searches back into global search with metadata', async ({ page }) => {
|
|
const capturedRequests: Array<Record<string, unknown>> = [];
|
|
await page.route('**/search/query**', async (route) => {
|
|
const request = route.request().postDataJSON() as Record<string, unknown>;
|
|
capturedRequests.push(request);
|
|
|
|
const query = String(request['q'] ?? '').toLowerCase();
|
|
const response = query.includes('policy blockers for cve-2024-21626')
|
|
? policyBlockerResponse
|
|
: criticalFindingResponse;
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(response),
|
|
});
|
|
});
|
|
await mockChatConversation(page, {
|
|
content: 'CVE-2024-21626 is still gating release decisions and policy evidence should be checked.',
|
|
citations: [
|
|
{ type: 'finding', path: 'CVE-2024-21626', verified: true },
|
|
{ type: 'policy', path: 'POL-118', verified: true },
|
|
],
|
|
groundingScore: 0.96,
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await typeInSearch(page, 'critical findings');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
await page.locator('.search__experience-bar').getByRole('button', { name: 'Act' }).click();
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
await page.locator('.entity-card__action--ask-ai').first().click();
|
|
|
|
await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 });
|
|
await expect(page.locator('[data-next-step="policy"]')).toBeVisible({ timeout: 10_000 });
|
|
|
|
await page.locator('[data-next-step="policy"]').click();
|
|
|
|
await expect(page.locator('.assistant-drawer')).toBeHidden({ timeout: 10_000 });
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
await expect(page.locator('app-global-search input[type="text"]')).toHaveValue(/policy blockers for CVE-2024-21626/i);
|
|
|
|
const policyRequest = capturedRequests.find((request) =>
|
|
String(request['q'] ?? '').toLowerCase().includes('policy blockers for cve-2024-21626'));
|
|
const ambient = policyRequest?.['ambient'] as Record<string, unknown> | undefined;
|
|
const lastAction = ambient?.['lastAction'] as Record<string, unknown> | undefined;
|
|
|
|
expect(lastAction?.['action']).toBe('chat_next_step_policy');
|
|
expect(lastAction?.['source']).toBe('advisory_ai_chat');
|
|
expect(lastAction?.['domain']).toBe('policy');
|
|
});
|
|
});
|
|
|
|
async function mockSearchResponses(
|
|
page: Page,
|
|
resolve: (normalizedQuery: string) => unknown,
|
|
): Promise<void> {
|
|
await page.route('**/search/query**', async (route) => {
|
|
const body = route.request().postDataJSON() as Record<string, unknown>;
|
|
const query = String(body['q'] ?? '').toLowerCase();
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(resolve(query)),
|
|
});
|
|
});
|
|
}
|
|
|
|
async function mockChatConversation(
|
|
page: Page,
|
|
response: {
|
|
content: string;
|
|
citations: Array<{ type: string; path: string; verified: boolean }>;
|
|
groundingScore: number;
|
|
},
|
|
): Promise<void> {
|
|
await page.route('**/api/v1/advisory-ai/conversations', async (route) => {
|
|
if (route.request().method() !== 'POST') {
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([]),
|
|
});
|
|
}
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
conversationId: 'conv-quality-1',
|
|
tenantId: 'test-tenant',
|
|
userId: 'tester',
|
|
context: {},
|
|
turns: [],
|
|
createdAt: '2026-03-06T00:00:00.000Z',
|
|
updatedAt: '2026-03-06T00:00:00.000Z',
|
|
}),
|
|
});
|
|
});
|
|
|
|
await page.route('**/api/v1/advisory-ai/conversations/*/turns', async (route) => {
|
|
if (route.request().method() !== 'POST') {
|
|
return route.continue();
|
|
}
|
|
|
|
const events = [
|
|
'event: progress',
|
|
'data: {"stage":"searching"}',
|
|
'',
|
|
'event: token',
|
|
`data: ${JSON.stringify({ content: response.content })}`,
|
|
'',
|
|
...response.citations.flatMap((citation) => ([
|
|
'event: citation',
|
|
`data: ${JSON.stringify(citation)}`,
|
|
'',
|
|
])),
|
|
'event: done',
|
|
`data: ${JSON.stringify({ turnId: 'turn-quality-1', groundingScore: response.groundingScore })}`,
|
|
'',
|
|
].join('\n');
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
headers: {
|
|
'content-type': 'text/event-stream; charset=utf-8',
|
|
'cache-control': 'no-cache',
|
|
},
|
|
body: events,
|
|
});
|
|
});
|
|
}
|