Repair search result routing and advisory query ranking
This commit is contained in:
master
@@ -107,6 +107,11 @@ const suites = [
|
||||
script: 'live-frontdoor-unified-search-route-matrix.mjs',
|
||||
reportPath: path.join(outputDir, 'live-frontdoor-unified-search-route-matrix.json'),
|
||||
},
|
||||
{
|
||||
name: 'search-result-action-sweep',
|
||||
script: 'live-search-result-action-sweep.mjs',
|
||||
reportPath: path.join(outputDir, 'live-search-result-action-sweep.json'),
|
||||
},
|
||||
];
|
||||
|
||||
const failureCountKeys = new Set([
|
||||
|
||||
@@ -0,0 +1,493 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { chromium } from 'playwright';
|
||||
|
||||
import { authenticateFrontdoor, createAuthenticatedContext } from './live-frontdoor-auth.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const webRoot = path.resolve(__dirname, '..');
|
||||
const outputDir = path.join(webRoot, 'output', 'playwright');
|
||||
const outputPath = path.join(outputDir, 'live-search-result-action-sweep.json');
|
||||
const authStatePath = path.join(outputDir, 'live-search-result-action-sweep.state.json');
|
||||
const authReportPath = path.join(outputDir, 'live-search-result-action-sweep.auth.json');
|
||||
const baseUrl = process.env.STELLAOPS_FRONTDOOR_BASE_URL?.trim() || 'https://stella-ops.local';
|
||||
const scopeQuery = 'tenant=demo-prod®ions=us-east&environments=stage&timeWindow=7d';
|
||||
const searchContexts = [
|
||||
{
|
||||
label: 'mission-board-cve',
|
||||
route: '/mission-control/board',
|
||||
query: 'cve',
|
||||
requireFindingAction: true,
|
||||
requireVexAction: true,
|
||||
requireKnowledgeAction: true,
|
||||
requireKnowledgeCards: true,
|
||||
requireVexCards: true,
|
||||
requireCanonicalDocsRoute: true,
|
||||
},
|
||||
{
|
||||
label: 'triage-cve',
|
||||
route: '/security/triage?pivot=cve',
|
||||
query: 'cve',
|
||||
requireFindingAction: true,
|
||||
requireVexAction: false,
|
||||
requireKnowledgeAction: false,
|
||||
requireKnowledgeCards: false,
|
||||
requireVexCards: false,
|
||||
requireCanonicalDocsRoute: false,
|
||||
},
|
||||
{
|
||||
label: 'mission-board-api-operation',
|
||||
route: '/mission-control/board',
|
||||
query: 'scanner scans api',
|
||||
requireFindingAction: false,
|
||||
requireVexAction: false,
|
||||
requireKnowledgeAction: false,
|
||||
requireKnowledgeCards: false,
|
||||
requireVexCards: false,
|
||||
requireCanonicalDocsRoute: false,
|
||||
requireApiCopyCard: true,
|
||||
},
|
||||
];
|
||||
|
||||
function buildUrl(route) {
|
||||
const separator = route.includes('?') ? '&' : '?';
|
||||
return `${baseUrl}${route}${separator}${scopeQuery}`;
|
||||
}
|
||||
|
||||
async function settle(page, ms = 1500) {
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 15_000 }).catch(() => {});
|
||||
await page.waitForTimeout(ms);
|
||||
}
|
||||
|
||||
async function snapshot(page, label) {
|
||||
const domSnapshot = await page.evaluate(() => {
|
||||
const heading =
|
||||
document.querySelector('h1, h2, [data-testid="page-title"], .page-title')?.textContent ?? '';
|
||||
const alerts = Array.from(
|
||||
document.querySelectorAll('[role="alert"], .alert, .error-banner, .success-banner, .loading-text, .search__loading, .search__empty'),
|
||||
)
|
||||
.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim())
|
||||
.filter(Boolean)
|
||||
.slice(0, 10);
|
||||
|
||||
return {
|
||||
title: document.title ?? '',
|
||||
heading: heading.replace(/\s+/g, ' ').trim(),
|
||||
alerts,
|
||||
};
|
||||
}).catch(() => ({
|
||||
title: '',
|
||||
heading: '',
|
||||
alerts: [],
|
||||
}));
|
||||
|
||||
return {
|
||||
label,
|
||||
url: page.url(),
|
||||
title: domSnapshot.title,
|
||||
heading: domSnapshot.heading,
|
||||
alerts: domSnapshot.alerts,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForDestinationContent(page) {
|
||||
await settle(page, 1500);
|
||||
|
||||
if (!page.url().includes('/docs/')) {
|
||||
return;
|
||||
}
|
||||
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const main = document.querySelector('main');
|
||||
return typeof main?.textContent === 'string' && main.textContent.replace(/\s+/g, ' ').trim().length > 64;
|
||||
},
|
||||
undefined,
|
||||
{ timeout: 10_000 },
|
||||
).catch(() => {});
|
||||
}
|
||||
|
||||
async function waitForSearchResolution(page, timeoutMs = 15_000) {
|
||||
const startedAt = Date.now();
|
||||
let sawLoading = false;
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
const state = await page.evaluate(() => ({
|
||||
cardCount: document.querySelectorAll('.entity-card').length,
|
||||
loadingVisible: Array.from(document.querySelectorAll('.search__loading'))
|
||||
.some((node) => (node.textContent || '').trim().length > 0),
|
||||
emptyTexts: Array.from(document.querySelectorAll('.search__empty, .search__empty-state-copy'))
|
||||
.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim())
|
||||
.filter(Boolean),
|
||||
})).catch(() => ({
|
||||
cardCount: 0,
|
||||
loadingVisible: false,
|
||||
emptyTexts: [],
|
||||
}));
|
||||
|
||||
sawLoading ||= state.loadingVisible;
|
||||
|
||||
if (state.cardCount > 0) {
|
||||
return {
|
||||
resolved: 'cards',
|
||||
sawLoading,
|
||||
cardCount: state.cardCount,
|
||||
emptyTexts: state.emptyTexts,
|
||||
waitedMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
if (!state.loadingVisible && state.emptyTexts.length > 0) {
|
||||
return {
|
||||
resolved: 'empty',
|
||||
sawLoading,
|
||||
cardCount: 0,
|
||||
emptyTexts: state.emptyTexts,
|
||||
waitedMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
await page.waitForTimeout(250);
|
||||
}
|
||||
|
||||
return {
|
||||
resolved: 'timeout',
|
||||
sawLoading,
|
||||
cardCount: await page.locator('.entity-card').count().catch(() => 0),
|
||||
emptyTexts: await page.locator('.search__empty, .search__empty-state-copy').evaluateAll(
|
||||
(nodes) => nodes.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim()).filter(Boolean),
|
||||
).catch(() => []),
|
||||
waitedMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectCards(page) {
|
||||
return page.locator('.entity-card').evaluateAll((nodes) =>
|
||||
nodes.slice(0, 8).map((node) => ({
|
||||
title: node.querySelector('.entity-card__title')?.textContent?.trim() || '',
|
||||
domain: node.querySelector('.entity-card__badge')?.textContent?.trim() || '',
|
||||
snippet: node.querySelector('.entity-card__snippet')?.textContent?.replace(/\s+/g, ' ').trim() || '',
|
||||
actions: Array.from(node.querySelectorAll('.entity-card__action')).map((button) => ({
|
||||
label: button.textContent?.replace(/\s+/g, ' ').trim() || '',
|
||||
isPrimary: button.classList.contains('entity-card__action--primary'),
|
||||
})).filter((action) => action.label.length > 0),
|
||||
})),
|
||||
).catch(() => []);
|
||||
}
|
||||
|
||||
async function executePrimaryAction(page, predicateLabel) {
|
||||
const cards = page.locator('.entity-card');
|
||||
const count = await cards.count().catch(() => 0);
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const card = cards.nth(index);
|
||||
const domain = await card.locator('.entity-card__badge').textContent().then((text) => text?.trim() || '').catch(() => '');
|
||||
if (!predicateLabel.test(domain)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const actionButton = card.locator('.entity-card__action--primary').first();
|
||||
const actionLabel = await actionButton.textContent().then((text) => text?.replace(/\s+/g, ' ').trim() || '').catch(() => '');
|
||||
process.stdout.write(`[live-search-result-action-sweep] click domain=${domain} label="${actionLabel}" index=${index}\n`);
|
||||
await actionButton.click({ timeout: 10_000 }).catch(() => {});
|
||||
process.stdout.write(`[live-search-result-action-sweep] clicked domain=${domain} url=${page.url()}\n`);
|
||||
await waitForDestinationContent(page);
|
||||
process.stdout.write(`[live-search-result-action-sweep] settled domain=${domain} url=${page.url()}\n`);
|
||||
return {
|
||||
matchedDomain: domain,
|
||||
actionLabel,
|
||||
url: page.url(),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function runSearchContext(page, context) {
|
||||
const responses = [];
|
||||
const responseListener = async (response) => {
|
||||
if (!response.url().includes('/api/v1/search/query')) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await response.json();
|
||||
responses.push({
|
||||
status: response.status(),
|
||||
url: response.url(),
|
||||
cards: (body.cards ?? []).slice(0, 8).map((card) => ({
|
||||
title: card.title ?? '',
|
||||
domain: card.domain ?? '',
|
||||
actions: (card.actions ?? []).map((action) => ({
|
||||
label: action.label ?? '',
|
||||
actionType: action.actionType ?? '',
|
||||
route: action.route ?? '',
|
||||
})),
|
||||
})),
|
||||
diagnostics: body.diagnostics ?? null,
|
||||
});
|
||||
} catch {
|
||||
responses.push({
|
||||
status: response.status(),
|
||||
url: response.url(),
|
||||
cards: [],
|
||||
diagnostics: null,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
page.on('response', responseListener);
|
||||
try {
|
||||
process.stdout.write(`[live-search-result-action-sweep] ${context.label} goto ${context.route}\n`);
|
||||
await page.goto(buildUrl(context.route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
||||
await settle(page);
|
||||
|
||||
const input = page.locator('input[aria-label="Global search"]').first();
|
||||
await input.click({ timeout: 10_000 });
|
||||
await input.fill(context.query);
|
||||
|
||||
const resolution = await waitForSearchResolution(page);
|
||||
process.stdout.write(`[live-search-result-action-sweep] ${context.label} resolved=${resolution.resolved} cards=${resolution.cardCount} waitedMs=${resolution.waitedMs}\n`);
|
||||
const cards = await collectCards(page);
|
||||
const latestResponse = responses.at(-1) ?? null;
|
||||
|
||||
const result = {
|
||||
label: context.label,
|
||||
route: context.route,
|
||||
query: context.query,
|
||||
expectations: {
|
||||
requireFindingAction: context.requireFindingAction === true,
|
||||
requireVexAction: context.requireVexAction === true,
|
||||
requireKnowledgeAction: context.requireKnowledgeAction === true,
|
||||
requireKnowledgeCards: context.requireKnowledgeCards === true,
|
||||
requireVexCards: context.requireVexCards === true,
|
||||
requireCanonicalDocsRoute: context.requireCanonicalDocsRoute === true,
|
||||
requireApiCopyCard: context.requireApiCopyCard === true,
|
||||
},
|
||||
resolution,
|
||||
cards,
|
||||
latestResponse,
|
||||
topCard: cards[0] ?? null,
|
||||
baseSnapshot: await snapshot(page, `${context.label}:results`),
|
||||
findingAction: null,
|
||||
vexAction: null,
|
||||
knowledgeAction: null,
|
||||
};
|
||||
|
||||
if (cards.length > 0 && context.requireFindingAction) {
|
||||
process.stdout.write(`[live-search-result-action-sweep] ${context.label} finding-action\n`);
|
||||
await page.goto(buildUrl(context.route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
||||
await settle(page);
|
||||
await page.locator('input[aria-label="Global search"]').first().click({ timeout: 10_000 });
|
||||
await page.locator('input[aria-label="Global search"]').first().fill(context.query);
|
||||
await waitForSearchResolution(page);
|
||||
result.findingAction = await executePrimaryAction(page, /^Findings$/i);
|
||||
}
|
||||
|
||||
if (cards.length > 0 && context.requireVexAction) {
|
||||
process.stdout.write(`[live-search-result-action-sweep] ${context.label} vex-action\n`);
|
||||
await page.goto(buildUrl(context.route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
||||
await settle(page);
|
||||
await page.locator('input[aria-label="Global search"]').first().click({ timeout: 10_000 });
|
||||
await page.locator('input[aria-label="Global search"]').first().fill(context.query);
|
||||
await waitForSearchResolution(page);
|
||||
result.vexAction = await executePrimaryAction(page, /^VEX/i);
|
||||
}
|
||||
|
||||
if (cards.length > 0 && context.requireKnowledgeAction) {
|
||||
process.stdout.write(`[live-search-result-action-sweep] ${context.label} knowledge-action\n`);
|
||||
await page.goto(buildUrl(context.route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
||||
await settle(page);
|
||||
await page.locator('input[aria-label="Global search"]').first().click({ timeout: 10_000 });
|
||||
await page.locator('input[aria-label="Global search"]').first().fill(context.query);
|
||||
await waitForSearchResolution(page);
|
||||
result.knowledgeAction = await executePrimaryAction(page, /^Knowledge$/i);
|
||||
}
|
||||
|
||||
return result;
|
||||
} finally {
|
||||
page.off('response', responseListener);
|
||||
}
|
||||
}
|
||||
|
||||
function collectFailures(results) {
|
||||
const failures = [];
|
||||
|
||||
for (const result of results) {
|
||||
const expectations = result.expectations ?? {};
|
||||
|
||||
if (result.resolution.resolved !== 'cards') {
|
||||
failures.push(`${result.label}: query "${result.query}" did not resolve to visible result cards.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.resolution.sawLoading) {
|
||||
failures.push(`${result.label}: query "${result.query}" never showed a loading state before results.`);
|
||||
}
|
||||
|
||||
const topDomain = (result.topCard?.domain ?? '').toLowerCase();
|
||||
const topPrimary = result.topCard?.actions?.find((action) => action.isPrimary)?.label ?? '';
|
||||
if (result.query === 'cve' && (topDomain.includes('knowledge') || topDomain.includes('api') || /copy curl/i.test(topPrimary))) {
|
||||
failures.push(`${result.label}: generic "${result.query}" search still ranks a knowledge/API card first (${result.topCard?.title ?? 'unknown'}).`);
|
||||
}
|
||||
|
||||
const findingRoute = result.latestResponse?.cards?.find((card) => card.domain === 'findings')?.actions?.[0]?.route ?? '';
|
||||
if (expectations.requireFindingAction && !findingRoute.startsWith('/security/triage')) {
|
||||
failures.push(`${result.label}: findings result is missing the canonical triage route.`);
|
||||
}
|
||||
|
||||
const knowledgeRoute = result.latestResponse?.cards?.find((card) => card.domain === 'knowledge')?.actions?.[0]?.route ?? '';
|
||||
if (expectations.requireKnowledgeCards && !knowledgeRoute) {
|
||||
failures.push(`${result.label}: query "${result.query}" did not surface a knowledge result.`);
|
||||
}
|
||||
|
||||
if (expectations.requireVexCards && !result.latestResponse?.cards?.some((card) => card.domain === 'vex')) {
|
||||
failures.push(`${result.label}: query "${result.query}" did not surface a VEX result.`);
|
||||
}
|
||||
|
||||
if (knowledgeRoute && !knowledgeRoute.startsWith('/docs/')) {
|
||||
failures.push(`${result.label}: knowledge result route is not a docs route (${knowledgeRoute}).`);
|
||||
}
|
||||
|
||||
if (expectations.requireFindingAction && !result.findingAction?.url?.includes('/security/triage')) {
|
||||
failures.push(`${result.label}: primary finding action did not land on Security Triage.`);
|
||||
}
|
||||
|
||||
if (expectations.requireVexAction && !result.vexAction?.url?.includes('/security/advisories-vex')) {
|
||||
failures.push(`${result.label}: primary VEX action did not land on Advisories & VEX.`);
|
||||
}
|
||||
|
||||
if (expectations.requireKnowledgeAction && !result.knowledgeAction?.url?.includes('/docs/')) {
|
||||
failures.push(`${result.label}: primary knowledge action did not land on Documentation.`);
|
||||
}
|
||||
|
||||
if (expectations.requireCanonicalDocsRoute && result.knowledgeAction?.url?.includes('/docs/docs%2F')) {
|
||||
failures.push(`${result.label}: primary knowledge action stayed on a non-canonical docs route (${result.knowledgeAction.url}).`);
|
||||
}
|
||||
|
||||
if (expectations.requireApiCopyCard) {
|
||||
const apiCard = result.latestResponse?.cards?.find((card) =>
|
||||
card.actions?.[0]?.label === 'Copy Curl');
|
||||
if (!apiCard) {
|
||||
failures.push(`${result.label}: explicit API query "${result.query}" did not surface a copy-first API card.`);
|
||||
} else {
|
||||
const primaryAction = apiCard.actions[0];
|
||||
const secondaryAction = apiCard.actions[1];
|
||||
if (primaryAction.actionType !== 'copy' || primaryAction.route) {
|
||||
failures.push(`${result.label}: API card primary action is not a pure copy action.`);
|
||||
}
|
||||
if (secondaryAction?.label !== 'Copy Operation ID' || secondaryAction.actionType !== 'copy') {
|
||||
failures.push(`${result.label}: API card secondary action is not "Copy Operation ID".`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return failures;
|
||||
}
|
||||
|
||||
function attachRuntimeListeners(page, runtime) {
|
||||
page.on('console', (message) => {
|
||||
if (message.type() === 'error') {
|
||||
runtime.consoleErrors.push(message.text());
|
||||
}
|
||||
});
|
||||
|
||||
page.on('pageerror', (error) => {
|
||||
runtime.pageErrors.push(error.message);
|
||||
});
|
||||
|
||||
page.on('requestfailed', (request) => {
|
||||
const url = request.url();
|
||||
if (/\.(?:css|js|map|png|jpg|jpeg|svg|woff2?)(?:$|\?)/i.test(url)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const errorText = request.failure()?.errorText ?? 'unknown';
|
||||
if (errorText === 'net::ERR_ABORTED') {
|
||||
return;
|
||||
}
|
||||
|
||||
runtime.requestFailures.push(`${request.method()} ${url} ${errorText}`);
|
||||
});
|
||||
|
||||
page.on('response', (response) => {
|
||||
const url = response.url();
|
||||
if (!url.includes('/api/v1/search/query')) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status() >= 400) {
|
||||
runtime.responseErrors.push(`${response.status()} ${response.request().method()} ${url}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
|
||||
const browser = await chromium.launch({ headless: process.env.PLAYWRIGHT_HEADLESS !== 'false' });
|
||||
const auth = await authenticateFrontdoor({ statePath: authStatePath, reportPath: authReportPath });
|
||||
const context = await createAuthenticatedContext(browser, auth, { statePath: auth.statePath });
|
||||
|
||||
const runtime = {
|
||||
consoleErrors: [],
|
||||
pageErrors: [],
|
||||
requestFailures: [],
|
||||
responseErrors: [],
|
||||
};
|
||||
|
||||
try {
|
||||
const results = [];
|
||||
for (const contextConfig of searchContexts) {
|
||||
process.stdout.write(`[live-search-result-action-sweep] START ${contextConfig.label} query="${contextConfig.query}"\n`);
|
||||
const page = await context.newPage();
|
||||
attachRuntimeListeners(page, runtime);
|
||||
try {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
results.push(await runSearchContext(page, contextConfig));
|
||||
} finally {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await page.close().catch(() => {});
|
||||
}
|
||||
process.stdout.write(`[live-search-result-action-sweep] DONE ${contextConfig.label}\n`);
|
||||
}
|
||||
|
||||
const failures = collectFailures(results);
|
||||
const report = {
|
||||
generatedAtUtc: new Date().toISOString(),
|
||||
baseUrl,
|
||||
scopeQuery,
|
||||
results,
|
||||
runtime,
|
||||
failedCheckCount: failures.length,
|
||||
runtimeIssueCount:
|
||||
runtime.consoleErrors.length +
|
||||
runtime.pageErrors.length +
|
||||
runtime.requestFailures.length +
|
||||
runtime.responseErrors.length,
|
||||
failures,
|
||||
ok:
|
||||
failures.length === 0 &&
|
||||
runtime.consoleErrors.length === 0 &&
|
||||
runtime.pageErrors.length === 0 &&
|
||||
runtime.requestFailures.length === 0 &&
|
||||
runtime.responseErrors.length === 0,
|
||||
};
|
||||
|
||||
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
|
||||
if (!report.ok) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} finally {
|
||||
await context.close().catch(() => {});
|
||||
await browser.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
await main();
|
||||
Reference in New Issue
Block a user