240 lines
7.8 KiB
TypeScript
240 lines
7.8 KiB
TypeScript
import { expect, test, type Page } from '@playwright/test';
|
|
|
|
import {
|
|
buildResponse,
|
|
emptyResponse,
|
|
findingCard,
|
|
setupAuthenticatedSession,
|
|
setupBasicMocks,
|
|
typeInSearch,
|
|
waitForEntityCards,
|
|
waitForResults,
|
|
} from './unified-search-fixtures';
|
|
|
|
const groundedFindingResponse = {
|
|
query: 'critical findings',
|
|
topK: 10,
|
|
cards: [
|
|
findingCard({
|
|
cveId: 'CVE-2024-21626',
|
|
title: 'CVE-2024-21626 in api-gateway',
|
|
snippet: 'Reachable critical vulnerability detected in the current production path.',
|
|
severity: 'critical',
|
|
}),
|
|
],
|
|
synthesis: {
|
|
summary: 'A reachable critical finding is blocking the current workflow and policy review is warranted.',
|
|
template: 'finding_overview',
|
|
confidence: 'high',
|
|
sourceCount: 2,
|
|
domainsCovered: ['Findings', 'Policy'],
|
|
citations: [
|
|
{
|
|
index: 1,
|
|
entityKey: 'cve:CVE-2024-21626',
|
|
title: 'CVE-2024-21626 in api-gateway',
|
|
},
|
|
],
|
|
},
|
|
diagnostics: {
|
|
ftsMatches: 3,
|
|
vectorMatches: 1,
|
|
entityCardCount: 1,
|
|
durationMs: 33,
|
|
usedVector: true,
|
|
mode: 'hybrid',
|
|
},
|
|
};
|
|
|
|
const narrowedFindingResponse = buildResponse(
|
|
'Which CVE, workload, or package should I narrow this to?',
|
|
[
|
|
findingCard({
|
|
cveId: 'CVE-2023-38545',
|
|
title: 'CVE-2023-38545 in edge-router',
|
|
snippet: 'A narrowed finding result is now grounded and ready for review.',
|
|
severity: 'high',
|
|
}),
|
|
],
|
|
{
|
|
summary: 'Narrowing the question exposed a grounded finding answer.',
|
|
template: 'finding_overview',
|
|
confidence: 'high',
|
|
sourceCount: 1,
|
|
domainsCovered: ['Findings'],
|
|
},
|
|
);
|
|
|
|
test.describe('Unified Search - Self-Serve Answer Panel', () => {
|
|
test.beforeEach(async ({ page }) => {
|
|
await setupBasicMocks(page);
|
|
await setupAuthenticatedSession(page);
|
|
});
|
|
|
|
test('shows page-owned common questions and a grounded answer panel', async ({ page }) => {
|
|
await mockSearchResponses(page, (query) =>
|
|
query.includes('critical findings') ? groundedFindingResponse : emptyResponse(query));
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
await expect(page.locator('[data-common-question]')).toContainText([
|
|
'Why is this exploitable in my environment?',
|
|
'What evidence blocks this release?',
|
|
'What is the safest remediation path?',
|
|
]);
|
|
|
|
await typeInSearch(page, 'critical findings');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
|
|
const answerPanel = page.locator('[data-answer-status="grounded"]');
|
|
await expect(answerPanel).toBeVisible();
|
|
await expect(answerPanel).toContainText('What we found');
|
|
await expect(answerPanel).toContainText('A reachable critical finding is blocking the current workflow and policy review is warranted.');
|
|
await expect(answerPanel).toContainText('Grounded in 2 source(s) across Findings, Policy.');
|
|
await expect(answerPanel.locator('[data-answer-citation]')).toContainText(['CVE-2024-21626 in api-gateway']);
|
|
});
|
|
|
|
test('uses clarify questions to rerun search and recover a grounded answer', async ({ page }) => {
|
|
await mockSearchResponses(page, (query) => {
|
|
if (query.includes('which cve, workload, or package should i narrow this to?')) {
|
|
return narrowedFindingResponse;
|
|
}
|
|
|
|
if (query.includes('mystery issue')) {
|
|
return emptyResponse('mystery issue');
|
|
}
|
|
|
|
return emptyResponse(query);
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await typeInSearch(page, 'mystery issue');
|
|
await waitForResults(page);
|
|
await expect(page.locator('[data-answer-status="clarify"]')).toBeVisible();
|
|
|
|
await page.getByRole('button', { name: 'Which CVE, workload, or package should I narrow this to?' }).click();
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
|
|
await expect(page.locator('app-global-search input[type="text"]')).toHaveValue(
|
|
'Which CVE, workload, or package should I narrow this to?',
|
|
);
|
|
await expect(page.locator('[data-answer-status="grounded"]')).toContainText(
|
|
'Narrowing the question exposed a grounded finding answer.',
|
|
);
|
|
});
|
|
|
|
test('opens AdvisoryAI from the answer panel with mode-aware context', async ({ page }) => {
|
|
await mockSearchResponses(page, (query) =>
|
|
query.includes('critical findings') ? groundedFindingResponse : emptyResponse(query));
|
|
await mockChatConversation(page, {
|
|
content: 'I can expand the grounded answer, explain the evidence, and recommend the safest next step.',
|
|
citations: [{ type: 'finding', path: 'CVE-2024-21626', verified: true }],
|
|
groundingScore: 0.95,
|
|
});
|
|
|
|
await page.goto('/security/triage');
|
|
await expect(page.locator('aside.sidebar')).toBeVisible({ timeout: 15_000 });
|
|
|
|
await page.locator('app-global-search input[type="text"]').focus();
|
|
await waitForResults(page);
|
|
await page.locator('.search__experience-bar').getByRole('button', { name: 'Act' }).click();
|
|
await typeInSearch(page, 'critical findings');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 1);
|
|
|
|
await page.locator('[data-answer-action="ask-ai"]').click();
|
|
|
|
await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 });
|
|
await expect(page.locator('.chat-mode-btn--active')).toHaveText(/Act/i);
|
|
await expect(page.locator('.chat-message.user .message-body').first()).toContainText(/Expand the grounded answer/i);
|
|
await expect(page.locator('.chat-message.user .message-body').first()).toContainText(/critical findings/i);
|
|
});
|
|
});
|
|
|
|
async function mockSearchResponses(
|
|
page: Page,
|
|
resolve: (normalizedQuery: string) => unknown,
|
|
): Promise<void> {
|
|
await page.route('**/search/query**', async (route) => {
|
|
const body = route.request().postDataJSON() as Record<string, unknown>;
|
|
const query = String(body['q'] ?? '').toLowerCase();
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify(resolve(query)),
|
|
});
|
|
});
|
|
}
|
|
|
|
async function mockChatConversation(
|
|
page: Page,
|
|
response: {
|
|
content: string;
|
|
citations: Array<{ type: string; path: string; verified: boolean }>;
|
|
groundingScore: number;
|
|
},
|
|
): Promise<void> {
|
|
await page.route('**/api/v1/advisory-ai/conversations', async (route) => {
|
|
if (route.request().method() !== 'POST') {
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify([]),
|
|
});
|
|
}
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
contentType: 'application/json',
|
|
body: JSON.stringify({
|
|
conversationId: 'conv-self-serve-1',
|
|
tenantId: 'test-tenant',
|
|
userId: 'tester',
|
|
context: {},
|
|
turns: [],
|
|
createdAt: '2026-03-07T00:00:00.000Z',
|
|
updatedAt: '2026-03-07T00:00:00.000Z',
|
|
}),
|
|
});
|
|
});
|
|
|
|
await page.route('**/api/v1/advisory-ai/conversations/*/turns', async (route) => {
|
|
if (route.request().method() !== 'POST') {
|
|
return route.continue();
|
|
}
|
|
|
|
const events = [
|
|
'event: progress',
|
|
'data: {"stage":"searching"}',
|
|
'',
|
|
'event: token',
|
|
`data: ${JSON.stringify({ content: response.content })}`,
|
|
'',
|
|
...response.citations.flatMap((citation) => ([
|
|
'event: citation',
|
|
`data: ${JSON.stringify(citation)}`,
|
|
'',
|
|
])),
|
|
'event: done',
|
|
`data: ${JSON.stringify({ turnId: 'turn-self-serve-1', groundingScore: response.groundingScore })}`,
|
|
'',
|
|
].join('\n');
|
|
|
|
return route.fulfill({
|
|
status: 200,
|
|
headers: {
|
|
'content-type': 'text/event-stream; charset=utf-8',
|
|
'cache-control': 'no-cache',
|
|
},
|
|
body: events,
|
|
});
|
|
});
|
|
}
|