520 lines
19 KiB
JavaScript
520 lines
19 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
import { chromium } from 'playwright';
|
|
|
|
import { authenticateFrontdoor, createAuthenticatedContext } from './live-frontdoor-auth.mjs';
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = path.dirname(__filename);
|
|
const webRoot = path.resolve(__dirname, '..');
|
|
const outputDir = path.join(webRoot, 'output', 'playwright');
|
|
const outputPath = path.join(outputDir, 'live-search-result-action-sweep.json');
|
|
const authStatePath = path.join(outputDir, 'live-search-result-action-sweep.state.json');
|
|
const authReportPath = path.join(outputDir, 'live-search-result-action-sweep.auth.json');
|
|
const baseUrl = process.env.STELLAOPS_FRONTDOOR_BASE_URL?.trim() || 'https://stella-ops.local';
|
|
const scopeQuery = 'tenant=demo-prod®ions=us-east&environments=stage&timeWindow=7d';
|
|
const searchContexts = [
|
|
{
|
|
label: 'mission-board-cve',
|
|
route: '/mission-control/board',
|
|
query: 'cve',
|
|
requireFindingAction: true,
|
|
requireVexAction: true,
|
|
requireKnowledgeAction: true,
|
|
requireKnowledgeCards: true,
|
|
requireVexCards: true,
|
|
requireCanonicalDocsRoute: true,
|
|
},
|
|
{
|
|
label: 'triage-cve',
|
|
route: '/security/triage?pivot=cve',
|
|
query: 'cve',
|
|
requireFindingAction: true,
|
|
requireVexAction: false,
|
|
requireKnowledgeAction: false,
|
|
requireKnowledgeCards: false,
|
|
requireVexCards: false,
|
|
requireCanonicalDocsRoute: false,
|
|
},
|
|
{
|
|
label: 'mission-board-api-operation',
|
|
route: '/mission-control/board',
|
|
query: 'scanner scans api',
|
|
requireFindingAction: false,
|
|
requireVexAction: false,
|
|
requireKnowledgeAction: false,
|
|
requireKnowledgeCards: false,
|
|
requireVexCards: false,
|
|
requireCanonicalDocsRoute: false,
|
|
requireApiCopyCard: true,
|
|
},
|
|
];
|
|
|
|
function buildUrl(route) {
|
|
const separator = route.includes('?') ? '&' : '?';
|
|
return `${baseUrl}${route}${separator}${scopeQuery}`;
|
|
}
|
|
|
|
async function settle(page, ms = 1500) {
|
|
await page.waitForLoadState('domcontentloaded', { timeout: 15_000 }).catch(() => {});
|
|
await page.waitForTimeout(ms);
|
|
}
|
|
|
|
async function snapshot(page, label) {
|
|
const domSnapshot = await page.evaluate(() => {
|
|
const heading =
|
|
document.querySelector('h1, h2, [data-testid="page-title"], .page-title')?.textContent ?? '';
|
|
const alerts = Array.from(
|
|
document.querySelectorAll('[role="alert"], .alert, .error-banner, .success-banner, .loading-text, .search__loading, .search__empty'),
|
|
)
|
|
.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim())
|
|
.filter(Boolean)
|
|
.slice(0, 10);
|
|
|
|
return {
|
|
title: document.title ?? '',
|
|
heading: heading.replace(/\s+/g, ' ').trim(),
|
|
alerts,
|
|
};
|
|
}).catch(() => ({
|
|
title: '',
|
|
heading: '',
|
|
alerts: [],
|
|
}));
|
|
|
|
return {
|
|
label,
|
|
url: page.url(),
|
|
title: domSnapshot.title,
|
|
heading: domSnapshot.heading,
|
|
alerts: domSnapshot.alerts,
|
|
};
|
|
}
|
|
|
|
async function waitForDestinationContent(page) {
|
|
await settle(page, 1500);
|
|
|
|
if (!page.url().includes('/docs/')) {
|
|
return {
|
|
docsContentLoaded: true,
|
|
docsContentPreview: '',
|
|
};
|
|
}
|
|
|
|
const docsContentLoaded = await page.waitForFunction(
|
|
() => {
|
|
const docsContent = document.querySelector('.docs-viewer__content, [data-testid="docs-content"]');
|
|
const text = typeof docsContent?.textContent === 'string'
|
|
? docsContent.textContent.replace(/\s+/g, ' ').trim()
|
|
: '';
|
|
return text.length > 64;
|
|
},
|
|
undefined,
|
|
{ timeout: 10_000 },
|
|
).then(() => true).catch(() => false);
|
|
|
|
const docsContentPreview = await page.locator('.docs-viewer__content, [data-testid="docs-content"]').first()
|
|
.textContent()
|
|
.then((text) => text?.replace(/\s+/g, ' ').trim().slice(0, 240) ?? '')
|
|
.catch(() => '');
|
|
|
|
return {
|
|
docsContentLoaded,
|
|
docsContentPreview,
|
|
};
|
|
}
|
|
|
|
async function waitForSearchResolution(page, timeoutMs = 15_000) {
|
|
const startedAt = Date.now();
|
|
let sawLoading = false;
|
|
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
const state = await page.evaluate(() => ({
|
|
cardCount: document.querySelectorAll('.entity-card').length,
|
|
loadingVisible: Array.from(document.querySelectorAll('.search__loading'))
|
|
.some((node) => (node.textContent || '').trim().length > 0),
|
|
emptyTexts: Array.from(document.querySelectorAll('.search__empty, .search__empty-state-copy'))
|
|
.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim())
|
|
.filter(Boolean),
|
|
})).catch(() => ({
|
|
cardCount: 0,
|
|
loadingVisible: false,
|
|
emptyTexts: [],
|
|
}));
|
|
|
|
sawLoading ||= state.loadingVisible;
|
|
|
|
if (state.cardCount > 0) {
|
|
return {
|
|
resolved: 'cards',
|
|
sawLoading,
|
|
cardCount: state.cardCount,
|
|
emptyTexts: state.emptyTexts,
|
|
waitedMs: Date.now() - startedAt,
|
|
};
|
|
}
|
|
|
|
if (!state.loadingVisible && state.emptyTexts.length > 0) {
|
|
return {
|
|
resolved: 'empty',
|
|
sawLoading,
|
|
cardCount: 0,
|
|
emptyTexts: state.emptyTexts,
|
|
waitedMs: Date.now() - startedAt,
|
|
};
|
|
}
|
|
|
|
await page.waitForTimeout(250);
|
|
}
|
|
|
|
return {
|
|
resolved: 'timeout',
|
|
sawLoading,
|
|
cardCount: await page.locator('.entity-card').count().catch(() => 0),
|
|
emptyTexts: await page.locator('.search__empty, .search__empty-state-copy').evaluateAll(
|
|
(nodes) => nodes.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim()).filter(Boolean),
|
|
).catch(() => []),
|
|
waitedMs: Date.now() - startedAt,
|
|
};
|
|
}
|
|
|
|
async function collectCards(page) {
|
|
return page.locator('.entity-card').evaluateAll((nodes) =>
|
|
nodes.slice(0, 8).map((node) => ({
|
|
title: node.querySelector('.entity-card__title')?.textContent?.trim() || '',
|
|
domain: node.querySelector('.entity-card__badge')?.textContent?.trim() || '',
|
|
snippet: node.querySelector('.entity-card__snippet')?.textContent?.replace(/\s+/g, ' ').trim() || '',
|
|
actions: Array.from(node.querySelectorAll('.entity-card__action')).map((button) => ({
|
|
label: button.textContent?.replace(/\s+/g, ' ').trim() || '',
|
|
isPrimary: button.classList.contains('entity-card__action--primary'),
|
|
})).filter((action) => action.label.length > 0),
|
|
})),
|
|
).catch(() => []);
|
|
}
|
|
|
|
async function executePrimaryAction(page, predicateLabel) {
|
|
const cards = page.locator('.entity-card');
|
|
const count = await cards.count().catch(() => 0);
|
|
for (let index = 0; index < count; index += 1) {
|
|
const card = cards.nth(index);
|
|
const domain = await card.locator('.entity-card__badge').textContent().then((text) => text?.trim() || '').catch(() => '');
|
|
if (!predicateLabel.test(domain)) {
|
|
continue;
|
|
}
|
|
|
|
const actionButton = card.locator('.entity-card__action--primary').first();
|
|
const actionLabel = await actionButton.textContent().then((text) => text?.replace(/\s+/g, ' ').trim() || '').catch(() => '');
|
|
process.stdout.write(`[live-search-result-action-sweep] click domain=${domain} label="${actionLabel}" index=${index}\n`);
|
|
await actionButton.click({ timeout: 10_000 }).catch(() => {});
|
|
process.stdout.write(`[live-search-result-action-sweep] clicked domain=${domain} url=${page.url()}\n`);
|
|
const destination = await waitForDestinationContent(page);
|
|
process.stdout.write(`[live-search-result-action-sweep] settled domain=${domain} url=${page.url()}\n`);
|
|
return {
|
|
matchedDomain: domain,
|
|
actionLabel,
|
|
url: page.url(),
|
|
destination,
|
|
snapshot: await snapshot(page, `${domain}:destination`),
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async function runSearchContext(page, context) {
|
|
const responses = [];
|
|
const responseListener = async (response) => {
|
|
if (!response.url().includes('/api/v1/search/query')) {
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const body = await response.json();
|
|
responses.push({
|
|
status: response.status(),
|
|
url: response.url(),
|
|
cards: (body.cards ?? []).slice(0, 8).map((card) => ({
|
|
title: card.title ?? '',
|
|
domain: card.domain ?? '',
|
|
actions: (card.actions ?? []).map((action) => ({
|
|
label: action.label ?? '',
|
|
actionType: action.actionType ?? '',
|
|
route: action.route ?? '',
|
|
})),
|
|
})),
|
|
diagnostics: body.diagnostics ?? null,
|
|
});
|
|
} catch {
|
|
responses.push({
|
|
status: response.status(),
|
|
url: response.url(),
|
|
cards: [],
|
|
diagnostics: null,
|
|
});
|
|
}
|
|
};
|
|
|
|
page.on('response', responseListener);
|
|
try {
|
|
process.stdout.write(`[live-search-result-action-sweep] ${context.label} goto ${context.route}\n`);
|
|
await page.goto(buildUrl(context.route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
|
await settle(page);
|
|
|
|
const input = page.locator('input[aria-label="Global search"]').first();
|
|
await input.click({ timeout: 10_000 });
|
|
await input.fill(context.query);
|
|
|
|
const resolution = await waitForSearchResolution(page);
|
|
process.stdout.write(`[live-search-result-action-sweep] ${context.label} resolved=${resolution.resolved} cards=${resolution.cardCount} waitedMs=${resolution.waitedMs}\n`);
|
|
const cards = await collectCards(page);
|
|
const latestResponse = responses.at(-1) ?? null;
|
|
|
|
const result = {
|
|
label: context.label,
|
|
route: context.route,
|
|
query: context.query,
|
|
expectations: {
|
|
requireFindingAction: context.requireFindingAction === true,
|
|
requireVexAction: context.requireVexAction === true,
|
|
requireKnowledgeAction: context.requireKnowledgeAction === true,
|
|
requireKnowledgeCards: context.requireKnowledgeCards === true,
|
|
requireVexCards: context.requireVexCards === true,
|
|
requireCanonicalDocsRoute: context.requireCanonicalDocsRoute === true,
|
|
requireApiCopyCard: context.requireApiCopyCard === true,
|
|
},
|
|
resolution,
|
|
cards,
|
|
latestResponse,
|
|
topCard: cards[0] ?? null,
|
|
baseSnapshot: await snapshot(page, `${context.label}:results`),
|
|
findingAction: null,
|
|
vexAction: null,
|
|
knowledgeAction: null,
|
|
};
|
|
|
|
if (cards.length > 0 && context.requireFindingAction) {
|
|
process.stdout.write(`[live-search-result-action-sweep] ${context.label} finding-action\n`);
|
|
await page.goto(buildUrl(context.route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
|
await settle(page);
|
|
await page.locator('input[aria-label="Global search"]').first().click({ timeout: 10_000 });
|
|
await page.locator('input[aria-label="Global search"]').first().fill(context.query);
|
|
await waitForSearchResolution(page);
|
|
result.findingAction = await executePrimaryAction(page, /^Findings$/i);
|
|
}
|
|
|
|
if (cards.length > 0 && context.requireVexAction) {
|
|
process.stdout.write(`[live-search-result-action-sweep] ${context.label} vex-action\n`);
|
|
await page.goto(buildUrl(context.route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
|
await settle(page);
|
|
await page.locator('input[aria-label="Global search"]').first().click({ timeout: 10_000 });
|
|
await page.locator('input[aria-label="Global search"]').first().fill(context.query);
|
|
await waitForSearchResolution(page);
|
|
result.vexAction = await executePrimaryAction(page, /^VEX/i);
|
|
}
|
|
|
|
if (cards.length > 0 && context.requireKnowledgeAction) {
|
|
process.stdout.write(`[live-search-result-action-sweep] ${context.label} knowledge-action\n`);
|
|
await page.goto(buildUrl(context.route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
|
await settle(page);
|
|
await page.locator('input[aria-label="Global search"]').first().click({ timeout: 10_000 });
|
|
await page.locator('input[aria-label="Global search"]').first().fill(context.query);
|
|
await waitForSearchResolution(page);
|
|
result.knowledgeAction = await executePrimaryAction(page, /^Knowledge$/i);
|
|
}
|
|
|
|
return result;
|
|
} finally {
|
|
page.off('response', responseListener);
|
|
}
|
|
}
|
|
|
|
function collectFailures(results) {
|
|
const failures = [];
|
|
|
|
for (const result of results) {
|
|
const expectations = result.expectations ?? {};
|
|
|
|
if (result.resolution.resolved !== 'cards') {
|
|
failures.push(`${result.label}: query "${result.query}" did not resolve to visible result cards.`);
|
|
continue;
|
|
}
|
|
|
|
if (!result.resolution.sawLoading) {
|
|
failures.push(`${result.label}: query "${result.query}" never showed a loading state before results.`);
|
|
}
|
|
|
|
const topDomain = (result.topCard?.domain ?? '').toLowerCase();
|
|
const topPrimary = result.topCard?.actions?.find((action) => action.isPrimary)?.label ?? '';
|
|
if (result.query === 'cve' && (topDomain.includes('knowledge') || topDomain.includes('api') || /copy curl/i.test(topPrimary))) {
|
|
failures.push(`${result.label}: generic "${result.query}" search still ranks a knowledge/API card first (${result.topCard?.title ?? 'unknown'}).`);
|
|
}
|
|
|
|
const findingRoute = result.latestResponse?.cards?.find((card) => card.domain === 'findings')?.actions?.[0]?.route ?? '';
|
|
if (expectations.requireFindingAction && !findingRoute.startsWith('/security/triage')) {
|
|
failures.push(`${result.label}: findings result is missing the canonical triage route.`);
|
|
}
|
|
|
|
const knowledgeRoute = result.latestResponse?.cards?.find((card) => card.domain === 'knowledge')?.actions?.[0]?.route ?? '';
|
|
if (expectations.requireKnowledgeCards && !knowledgeRoute) {
|
|
failures.push(`${result.label}: query "${result.query}" did not surface a knowledge result.`);
|
|
}
|
|
|
|
if (expectations.requireVexCards && !result.latestResponse?.cards?.some((card) => card.domain === 'vex')) {
|
|
failures.push(`${result.label}: query "${result.query}" did not surface a VEX result.`);
|
|
}
|
|
|
|
if (knowledgeRoute && !knowledgeRoute.startsWith('/docs/')) {
|
|
failures.push(`${result.label}: knowledge result route is not a docs route (${knowledgeRoute}).`);
|
|
}
|
|
|
|
if (expectations.requireFindingAction && !result.findingAction?.url?.includes('/security/triage')) {
|
|
failures.push(`${result.label}: primary finding action did not land on Security Triage.`);
|
|
}
|
|
|
|
if (expectations.requireVexAction && !result.vexAction?.url?.includes('/security/advisories-vex')) {
|
|
failures.push(`${result.label}: primary VEX action did not land on Advisories & VEX.`);
|
|
}
|
|
|
|
if (expectations.requireKnowledgeAction && !result.knowledgeAction?.url?.includes('/docs/')) {
|
|
failures.push(`${result.label}: primary knowledge action did not land on Documentation.`);
|
|
}
|
|
|
|
if (expectations.requireCanonicalDocsRoute && result.knowledgeAction?.url?.includes('/docs/docs%2F')) {
|
|
failures.push(`${result.label}: primary knowledge action stayed on a non-canonical docs route (${result.knowledgeAction.url}).`);
|
|
}
|
|
|
|
if (
|
|
expectations.requireKnowledgeAction &&
|
|
result.knowledgeAction?.url?.includes('/docs/') &&
|
|
result.knowledgeAction?.destination?.docsContentLoaded !== true
|
|
) {
|
|
failures.push(`${result.label}: primary knowledge action landed on a docs route without rendered documentation content.`);
|
|
}
|
|
|
|
if (expectations.requireApiCopyCard) {
|
|
const apiCard = result.latestResponse?.cards?.find((card) =>
|
|
card.actions?.[0]?.label === 'Copy Curl');
|
|
if (!apiCard) {
|
|
failures.push(`${result.label}: explicit API query "${result.query}" did not surface a copy-first API card.`);
|
|
} else {
|
|
const primaryAction = apiCard.actions[0];
|
|
const secondaryAction = apiCard.actions[1];
|
|
if (primaryAction.actionType !== 'copy' || primaryAction.route) {
|
|
failures.push(`${result.label}: API card primary action is not a pure copy action.`);
|
|
}
|
|
if (secondaryAction?.label !== 'Copy Operation ID' || secondaryAction.actionType !== 'copy') {
|
|
failures.push(`${result.label}: API card secondary action is not "Copy Operation ID".`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return failures;
|
|
}
|
|
|
|
function attachRuntimeListeners(page, runtime) {
|
|
page.on('console', (message) => {
|
|
if (message.type() === 'error') {
|
|
runtime.consoleErrors.push(message.text());
|
|
}
|
|
});
|
|
|
|
page.on('pageerror', (error) => {
|
|
runtime.pageErrors.push(error.message);
|
|
});
|
|
|
|
page.on('requestfailed', (request) => {
|
|
const url = request.url();
|
|
if (/\.(?:css|js|map|png|jpg|jpeg|svg|woff2?)(?:$|\?)/i.test(url)) {
|
|
return;
|
|
}
|
|
|
|
const errorText = request.failure()?.errorText ?? 'unknown';
|
|
if (errorText === 'net::ERR_ABORTED') {
|
|
return;
|
|
}
|
|
|
|
runtime.requestFailures.push(`${request.method()} ${url} ${errorText}`);
|
|
});
|
|
|
|
page.on('response', (response) => {
|
|
const url = response.url();
|
|
if (!url.includes('/api/v1/search/query')) {
|
|
return;
|
|
}
|
|
|
|
if (response.status() >= 400) {
|
|
runtime.responseErrors.push(`${response.status()} ${response.request().method()} ${url}`);
|
|
}
|
|
});
|
|
}
|
|
|
|
async function main() {
|
|
await mkdir(outputDir, { recursive: true });
|
|
|
|
const browser = await chromium.launch({ headless: process.env.PLAYWRIGHT_HEADLESS !== 'false' });
|
|
const auth = await authenticateFrontdoor({ statePath: authStatePath, reportPath: authReportPath });
|
|
const context = await createAuthenticatedContext(browser, auth, { statePath: auth.statePath });
|
|
|
|
const runtime = {
|
|
consoleErrors: [],
|
|
pageErrors: [],
|
|
requestFailures: [],
|
|
responseErrors: [],
|
|
};
|
|
|
|
try {
|
|
const results = [];
|
|
for (const contextConfig of searchContexts) {
|
|
process.stdout.write(`[live-search-result-action-sweep] START ${contextConfig.label} query="${contextConfig.query}"\n`);
|
|
const page = await context.newPage();
|
|
attachRuntimeListeners(page, runtime);
|
|
try {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
results.push(await runSearchContext(page, contextConfig));
|
|
} finally {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
await page.close().catch(() => {});
|
|
}
|
|
process.stdout.write(`[live-search-result-action-sweep] DONE ${contextConfig.label}\n`);
|
|
}
|
|
|
|
const failures = collectFailures(results);
|
|
const report = {
|
|
generatedAtUtc: new Date().toISOString(),
|
|
baseUrl,
|
|
scopeQuery,
|
|
results,
|
|
runtime,
|
|
failedCheckCount: failures.length,
|
|
runtimeIssueCount:
|
|
runtime.consoleErrors.length +
|
|
runtime.pageErrors.length +
|
|
runtime.requestFailures.length +
|
|
runtime.responseErrors.length,
|
|
failures,
|
|
ok:
|
|
failures.length === 0 &&
|
|
runtime.consoleErrors.length === 0 &&
|
|
runtime.pageErrors.length === 0 &&
|
|
runtime.requestFailures.length === 0 &&
|
|
runtime.responseErrors.length === 0,
|
|
};
|
|
|
|
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
|
|
if (!report.ok) {
|
|
process.exitCode = 1;
|
|
}
|
|
} finally {
|
|
await context.close().catch(() => {});
|
|
await browser.close().catch(() => {});
|
|
}
|
|
}
|
|
|
|
await main();
|