Refine live Playwright changed-surface checks
This commit is contained in:
master
@@ -0,0 +1,457 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { chromium } from 'playwright';
|
||||
|
||||
import { authenticateFrontdoor } from './live-frontdoor-auth.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const webRoot = path.resolve(__dirname, '..');
|
||||
const outputDirectory = path.join(webRoot, 'output', 'playwright');
|
||||
|
||||
const baseUrl = process.env.STELLAOPS_FRONTDOOR_BASE_URL?.trim() || 'https://stella-ops.local';
|
||||
const statePath = path.join(outputDirectory, 'live-frontdoor-auth-state.json');
|
||||
const outputPath = path.join(outputDirectory, 'live-frontdoor-changed-surfaces.json');
|
||||
|
||||
const surfaceConfigs = [
|
||||
{
|
||||
key: 'mission-board',
|
||||
path: '/mission-control/board?tenant=demo-prod®ions=us-east&environments=stage&timeWindow=7d',
|
||||
heading: /dashboard|mission board|mission control/i,
|
||||
searchQuery: 'database connectivity',
|
||||
actions: [
|
||||
{
|
||||
key: 'sbom-lake',
|
||||
selector: '#main-content a[href*="/security/sbom-lake"], #main-content a:has-text("View SBOM")',
|
||||
expectedUrlPattern: '/security/sbom-lake',
|
||||
expectedTextPattern: /sbom lake/i,
|
||||
},
|
||||
{
|
||||
key: 'reachability',
|
||||
selector: '#main-content a[href*="/security/reachability"], #main-content a:has-text("View reachability")',
|
||||
expectedUrlPattern: '/security/reachability',
|
||||
expectedTextPattern: /reachability/i,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'advisories-vex',
|
||||
path: '/security/advisories-vex',
|
||||
heading: /advisories|vex/i,
|
||||
searchQuery: 'what evidence conflicts with this vex',
|
||||
},
|
||||
{
|
||||
key: 'policy-overview',
|
||||
path: '/ops/policy',
|
||||
heading: /policy/i,
|
||||
searchQuery: 'production deny rules',
|
||||
},
|
||||
{
|
||||
key: 'evidence-threads',
|
||||
path: '/evidence/threads',
|
||||
heading: /evidence threads/i,
|
||||
searchQuery: 'artifact digest',
|
||||
actions: [
|
||||
{
|
||||
key: 'search-empty-result',
|
||||
fillSelector: 'main input[placeholder*="pkg:oci"], main input[matinput]',
|
||||
fillValue: 'pkg:npm/example@1.0.0',
|
||||
submitSelector: 'main button:has-text("Search")',
|
||||
expectedUrlPattern: '/evidence/threads',
|
||||
expectedTextPattern: /no evidence threads matched this package url/i,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'evidence-thread-detail-missing',
|
||||
path: '/evidence/threads/missing-demo-canonical?purl=pkg%3Anpm%2Fexample%401.0.0',
|
||||
heading: /missing-demo-canonical/i,
|
||||
searchQuery: 'artifact digest',
|
||||
allowedConsolePatterns: [
|
||||
/Failed to load resource: the server responded with a status of 404 \(\)/i,
|
||||
],
|
||||
allowedResponsePatterns: [
|
||||
/\/api\/v1\/evidence\/thread\/missing-demo-canonical$/,
|
||||
],
|
||||
actions: [
|
||||
{
|
||||
key: 'back-to-search',
|
||||
selector: 'main button:has-text("Back to Search")',
|
||||
expectedUrlPattern: '/evidence/threads',
|
||||
expectedTextPattern: /evidence threads/i,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'release-investigation-timeline',
|
||||
path: '/releases/investigation/timeline',
|
||||
heading: /timeline/i,
|
||||
searchQuery: 'release correlation timeline',
|
||||
},
|
||||
{
|
||||
key: 'release-investigation-deploy-diff',
|
||||
path: '/releases/investigation/deploy-diff',
|
||||
heading: /deploy diff|deployment diff|missing parameters/i,
|
||||
searchQuery: 'deployment diff',
|
||||
},
|
||||
{
|
||||
key: 'release-investigation-change-trace',
|
||||
path: '/releases/investigation/change-trace',
|
||||
heading: /change trace/i,
|
||||
searchQuery: 'change trace',
|
||||
},
|
||||
{
|
||||
key: 'registry-admin',
|
||||
path: '/ops/integrations/registry-admin',
|
||||
heading: /registry token service/i,
|
||||
searchQuery: 'registry token plan',
|
||||
actions: [
|
||||
{
|
||||
key: 'audit-tab',
|
||||
selector: 'a[href*="/registry-admin/audit"], button:has-text("Audit")',
|
||||
expectedUrlPattern: '/registry-admin/audit',
|
||||
expectedTextPattern: /audit/i,
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
function shouldIgnoreUrl(url) {
|
||||
return /\.(?:css|js|map|png|jpg|jpeg|svg|woff2?)(?:$|\?)/i.test(url)
|
||||
|| url.startsWith('data:');
|
||||
}
|
||||
|
||||
function trimText(value, maxLength = 400) {
|
||||
const normalized = value.replace(/\s+/g, ' ').trim();
|
||||
return normalized.length > maxLength ? `${normalized.slice(0, maxLength)}...` : normalized;
|
||||
}
|
||||
|
||||
async function collectHeadings(page) {
|
||||
return page.locator('h1, main h1, main h2, h2').evaluateAll((elements) =>
|
||||
elements
|
||||
.map((element) => (element.textContent || '').replace(/\s+/g, ' ').trim())
|
||||
.filter((text, index, values) => text.length > 0 && values.indexOf(text) === index),
|
||||
).catch(() => []);
|
||||
}
|
||||
|
||||
async function collectBodyText(page, maxLength = 1_200) {
|
||||
return trimText(await page.locator('body').innerText().catch(() => ''), maxLength);
|
||||
}
|
||||
|
||||
function matchesPattern(pattern, headings, bodyText = '') {
|
||||
if (!pattern) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return headings.some((heading) => pattern.test(heading)) || pattern.test(bodyText);
|
||||
}
|
||||
|
||||
function firstMatchingHeading(pattern, headings) {
|
||||
if (!pattern) {
|
||||
return headings[0] ?? '';
|
||||
}
|
||||
|
||||
return headings.find((heading) => pattern.test(heading)) ?? headings[0] ?? '';
|
||||
}
|
||||
|
||||
function matchesAnyPattern(patterns, value) {
|
||||
if (!Array.isArray(patterns) || patterns.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return patterns.some((pattern) => pattern.test(value));
|
||||
}
|
||||
|
||||
async function collectVisibleProblemTexts(page) {
|
||||
return page.locator([
|
||||
'[role="alert"]',
|
||||
'.alert',
|
||||
'.banner',
|
||||
'.status-banner',
|
||||
'.degraded-banner',
|
||||
'.warning-banner',
|
||||
'.error-banner',
|
||||
'.empty-state',
|
||||
'.error-state',
|
||||
].join(', ')).evaluateAll((elements) =>
|
||||
elements
|
||||
.map((element) => (element.textContent || '').replace(/\s+/g, ' ').trim())
|
||||
.filter((text) =>
|
||||
text.length > 0 && /(failed|unable|error|warning|degraded|unavailable|timed out|timeout|no results)/i.test(text),
|
||||
),
|
||||
).catch(() => []);
|
||||
}
|
||||
|
||||
async function runGlobalSearch(page, query, record) {
|
||||
const searchInput = page.locator('app-global-search input[type="text"]').first();
|
||||
const available = await searchInput.isVisible().catch(() => false);
|
||||
if (!available) {
|
||||
return { available: false };
|
||||
}
|
||||
|
||||
let queryResponse = null;
|
||||
let suggestionResponse = null;
|
||||
|
||||
const onResponse = async (response) => {
|
||||
const url = response.url();
|
||||
if (url.includes('/api/v1/search/query')) {
|
||||
queryResponse = {
|
||||
status: response.status(),
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
if (url.includes('/api/v1/search/suggestions/evaluate')) {
|
||||
suggestionResponse = {
|
||||
status: response.status(),
|
||||
url,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
page.on('response', onResponse);
|
||||
|
||||
await searchInput.click();
|
||||
await searchInput.fill(query);
|
||||
await page.waitForTimeout(1_800);
|
||||
|
||||
await page.locator('.search__results').waitFor({ state: 'visible', timeout: 15_000 }).catch(() => {});
|
||||
await page.waitForTimeout(2_000);
|
||||
page.off('response', onResponse);
|
||||
|
||||
const resultsVisible = await page.locator('.search__results').isVisible().catch(() => false);
|
||||
const resultsText = resultsVisible
|
||||
? trimText(await page.locator('.search__results').innerText().catch(() => ''))
|
||||
: '';
|
||||
const suggestionTexts = (await page.locator('.search__suggestions .search__chip').allTextContents().catch(() => []))
|
||||
.map((value) => trimText(value, 140))
|
||||
.filter((value) => value.length > 0);
|
||||
|
||||
record.searchRequests = {
|
||||
queryResponse,
|
||||
suggestionResponse,
|
||||
};
|
||||
|
||||
return {
|
||||
available: true,
|
||||
query,
|
||||
inputValue: await searchInput.inputValue().catch(() => ''),
|
||||
resultsVisible,
|
||||
suggestionCount: suggestionTexts.length,
|
||||
suggestions: suggestionTexts.slice(0, 6),
|
||||
resultsText,
|
||||
};
|
||||
}
|
||||
|
||||
async function inspectSurface(context, surface) {
|
||||
const page = await context.newPage();
|
||||
const record = {
|
||||
key: surface.key,
|
||||
url: `${baseUrl}${surface.path}`,
|
||||
finalUrl: '',
|
||||
title: '',
|
||||
headings: [],
|
||||
headingMatched: false,
|
||||
headingText: '',
|
||||
problemTexts: [],
|
||||
consoleErrors: [],
|
||||
pageErrors: [],
|
||||
requestFailures: [],
|
||||
responseErrors: [],
|
||||
searchRequests: null,
|
||||
search: null,
|
||||
actions: [],
|
||||
};
|
||||
|
||||
page.on('console', (message) => {
|
||||
if (message.type() === 'error') {
|
||||
const text = message.text();
|
||||
if (matchesAnyPattern(surface.allowedConsolePatterns, text)) {
|
||||
return;
|
||||
}
|
||||
|
||||
record.consoleErrors.push(text);
|
||||
}
|
||||
});
|
||||
|
||||
page.on('pageerror', (error) => {
|
||||
record.pageErrors.push(error.message);
|
||||
});
|
||||
|
||||
page.on('requestfailed', (request) => {
|
||||
const url = request.url();
|
||||
if (shouldIgnoreUrl(url)) {
|
||||
return;
|
||||
}
|
||||
|
||||
record.requestFailures.push({
|
||||
method: request.method(),
|
||||
url,
|
||||
error: request.failure()?.errorText ?? 'unknown',
|
||||
});
|
||||
});
|
||||
|
||||
page.on('response', (response) => {
|
||||
const url = response.url();
|
||||
if (shouldIgnoreUrl(url)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status() >= 400) {
|
||||
if (matchesAnyPattern(surface.allowedResponsePatterns, url)) {
|
||||
return;
|
||||
}
|
||||
|
||||
record.responseErrors.push({
|
||||
status: response.status(),
|
||||
method: response.request().method(),
|
||||
url,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await page.goto(record.url, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30_000,
|
||||
});
|
||||
await page.waitForTimeout(4_000);
|
||||
|
||||
record.finalUrl = page.url();
|
||||
record.title = await page.title();
|
||||
const headings = await collectHeadings(page);
|
||||
record.headings = headings.slice(0, 8);
|
||||
record.headingText = firstMatchingHeading(surface.heading, headings);
|
||||
record.headingMatched = matchesPattern(surface.heading, headings);
|
||||
record.problemTexts = await collectVisibleProblemTexts(page);
|
||||
record.search = await runGlobalSearch(page, surface.searchQuery, record);
|
||||
|
||||
const screenshotPath = path.join(outputDirectory, `${surface.key}.png`);
|
||||
await page.screenshot({ path: screenshotPath, fullPage: true }).catch(() => {});
|
||||
record.screenshotPath = screenshotPath;
|
||||
|
||||
await page.close();
|
||||
return record;
|
||||
}
|
||||
|
||||
async function verifySurfaceActions(context, surface) {
|
||||
if (!Array.isArray(surface.actions) || surface.actions.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const results = [];
|
||||
for (const action of surface.actions) {
|
||||
const page = await context.newPage();
|
||||
try {
|
||||
await page.goto(`${baseUrl}${surface.path}`, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 30_000,
|
||||
});
|
||||
await page.waitForTimeout(4_000);
|
||||
|
||||
if (action.fillSelector) {
|
||||
const fillTarget = page.locator(action.fillSelector).first();
|
||||
const fillVisible = await fillTarget.isVisible().catch(() => false);
|
||||
if (!fillVisible) {
|
||||
results.push({
|
||||
key: action.key,
|
||||
ok: false,
|
||||
reason: 'fill_target_not_visible',
|
||||
finalUrl: page.url(),
|
||||
});
|
||||
await page.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
await fillTarget.fill(action.fillValue ?? '');
|
||||
}
|
||||
|
||||
const triggerSelector = action.submitSelector ?? action.selector;
|
||||
const trigger = page.locator(triggerSelector).first();
|
||||
const visible = await trigger.isVisible().catch(() => false);
|
||||
if (!visible) {
|
||||
results.push({
|
||||
key: action.key,
|
||||
ok: false,
|
||||
reason: 'link_not_visible',
|
||||
finalUrl: page.url(),
|
||||
});
|
||||
await page.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
await trigger.click();
|
||||
await page.waitForTimeout(4_000);
|
||||
const headings = await collectHeadings(page);
|
||||
const bodyText = await collectBodyText(page);
|
||||
const headingText = firstMatchingHeading(action.expectedTextPattern, headings);
|
||||
const finalUrl = page.url();
|
||||
const ok = finalUrl.includes(action.expectedUrlPattern)
|
||||
&& matchesPattern(action.expectedTextPattern, headings, bodyText);
|
||||
|
||||
results.push({
|
||||
key: action.key,
|
||||
ok,
|
||||
finalUrl,
|
||||
headingText,
|
||||
});
|
||||
} finally {
|
||||
await page.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
mkdirSync(outputDirectory, { recursive: true });
|
||||
const authReport = await authenticateFrontdoor({
|
||||
statePath,
|
||||
reportPath: path.join(outputDirectory, 'live-frontdoor-auth-report.json'),
|
||||
});
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ['--disable-dev-shm-usage'],
|
||||
});
|
||||
const context = await browser.newContext({
|
||||
ignoreHTTPSErrors: true,
|
||||
storageState: statePath,
|
||||
});
|
||||
const sessionEntries = Array.isArray(authReport.storage?.sessionStorageEntries)
|
||||
? authReport.storage.sessionStorageEntries
|
||||
: [];
|
||||
await context.addInitScript((entries) => {
|
||||
for (const [key, value] of entries) {
|
||||
if (typeof key === 'string' && typeof value === 'string') {
|
||||
sessionStorage.setItem(key, value);
|
||||
}
|
||||
}
|
||||
}, sessionEntries);
|
||||
|
||||
const report = {
|
||||
generatedAtUtc: new Date().toISOString(),
|
||||
baseUrl,
|
||||
surfaces: [],
|
||||
};
|
||||
|
||||
for (const surface of surfaceConfigs) {
|
||||
const surfaceReport = await inspectSurface(context, surface);
|
||||
surfaceReport.actions = await verifySurfaceActions(context, surface);
|
||||
report.surfaces.push(surfaceReport);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === __filename) {
|
||||
main().catch((error) => {
|
||||
process.stderr.write(`[live-frontdoor-changed-surfaces] ${error instanceof Error ? error.message : String(error)}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user