Repair live jobs queues action handoffs
This commit is contained in:
master
287
src/Web/StellaOps.Web/scripts/live-jobs-queues-action-sweep.mjs
Normal file
287
src/Web/StellaOps.Web/scripts/live-jobs-queues-action-sweep.mjs
Normal file
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { chromium } from 'playwright';
|
||||
|
||||
import { authenticateFrontdoor, createAuthenticatedContext } from './live-frontdoor-auth.mjs';
|
||||
|
||||
const webRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const outputDir = path.join(webRoot, 'output', 'playwright');
|
||||
const outputPath = path.join(outputDir, 'live-jobs-queues-action-sweep.json');
|
||||
const authStatePath = path.join(outputDir, 'live-jobs-queues-action-sweep.state.json');
|
||||
const authReportPath = path.join(outputDir, 'live-jobs-queues-action-sweep.auth.json');
|
||||
const scopeQuery = 'tenant=demo-prod®ions=us-east&environments=stage&timeWindow=7d';
|
||||
const routePath = '/ops/operations/jobs-queues';
|
||||
|
||||
function scopedUrl(route = routePath) {
|
||||
const separator = route.includes('?') ? '&' : '?';
|
||||
return `https://stella-ops.local${route}${separator}${scopeQuery}`;
|
||||
}
|
||||
|
||||
async function settle(page) {
|
||||
await page.waitForLoadState('domcontentloaded', { timeout: 20_000 }).catch(() => {});
|
||||
await page.waitForTimeout(1_200);
|
||||
}
|
||||
|
||||
async function gotoPage(page) {
|
||||
await page.goto(scopedUrl(), { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
||||
await settle(page);
|
||||
}
|
||||
|
||||
async function clickTab(page, tabName) {
|
||||
await page.getByRole('button', { name: tabName }).click({ timeout: 10_000 });
|
||||
await settle(page);
|
||||
}
|
||||
|
||||
async function rowCount(page) {
|
||||
return page.locator('tbody tr').count();
|
||||
}
|
||||
|
||||
function searchFilter(page) {
|
||||
return page.locator('section.filters input[type="search"]').first();
|
||||
}
|
||||
|
||||
function filterSelect(page, index) {
|
||||
return page.locator('section.filters select').nth(index);
|
||||
}
|
||||
|
||||
function actionBanner(page) {
|
||||
return page.locator('section.jobs-queues .jobs-queues__banner[role="status"]').first();
|
||||
}
|
||||
|
||||
async function captureRuntimeIssues(page) {
|
||||
const alerts = await page
|
||||
.locator('[role="alert"], .error-banner, .toast, .notification')
|
||||
.evaluateAll((nodes) =>
|
||||
nodes
|
||||
.map((node) => (node.textContent || '').trim().replace(/\s+/g, ' '))
|
||||
.filter(Boolean),
|
||||
)
|
||||
.catch(() => []);
|
||||
|
||||
return alerts;
|
||||
}
|
||||
|
||||
async function readActionHrefs(page) {
|
||||
return page.locator('tbody tr:first-child .actions a').evaluateAll((links) =>
|
||||
links.map((link) => ({
|
||||
label: (link.textContent || '').trim().replace(/\s+/g, ' '),
|
||||
href: link.getAttribute('href') || '',
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
async function assert(condition, message, details = {}) {
|
||||
if (!condition) {
|
||||
throw new Error(`${message}${Object.keys(details).length ? ` ${JSON.stringify(details)}` : ''}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function run() {
|
||||
await mkdir(outputDir, { recursive: true });
|
||||
|
||||
const authReport = await authenticateFrontdoor({
|
||||
statePath: authStatePath,
|
||||
reportPath: authReportPath,
|
||||
headless: true,
|
||||
});
|
||||
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ['--disable-dev-shm-usage'],
|
||||
});
|
||||
|
||||
const context = await createAuthenticatedContext(browser, authReport, { statePath: authStatePath });
|
||||
const page = await context.newPage();
|
||||
const consoleErrors = [];
|
||||
const responseErrors = [];
|
||||
const requestFailures = [];
|
||||
|
||||
page.on('console', (message) => {
|
||||
if (message.type() === 'error') {
|
||||
consoleErrors.push(message.text());
|
||||
}
|
||||
});
|
||||
|
||||
page.on('response', (response) => {
|
||||
const url = response.url();
|
||||
if (/\.(?:css|js|map|png|jpg|jpeg|svg|woff2?)(?:$|\?)/i.test(url)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status() >= 400 && !url.includes('/connect/authorize')) {
|
||||
responseErrors.push({
|
||||
status: response.status(),
|
||||
method: response.request().method(),
|
||||
url,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
page.on('requestfailed', (request) => {
|
||||
const url = request.url();
|
||||
const failure = request.failure()?.errorText ?? 'unknown';
|
||||
if (failure === 'net::ERR_ABORTED') {
|
||||
return;
|
||||
}
|
||||
|
||||
if (/\.(?:css|js|map|png|jpg|jpeg|svg|woff2?)(?:$|\?)/i.test(url)) {
|
||||
return;
|
||||
}
|
||||
|
||||
requestFailures.push({
|
||||
method: request.method(),
|
||||
url,
|
||||
error: failure,
|
||||
});
|
||||
});
|
||||
|
||||
const checks = [];
|
||||
|
||||
try {
|
||||
await gotoPage(page);
|
||||
await assert((await page.locator('h1').first().textContent())?.trim() === 'Jobs & Queues', 'Unexpected page heading.');
|
||||
|
||||
let actions = await readActionHrefs(page);
|
||||
await assert(actions.length === 2, 'Jobs tab did not expose the expected two row actions.', { actions });
|
||||
await assert(actions[0].href.includes('/ops/operations/jobengine'), 'Jobs primary action did not target JobEngine.', { actions });
|
||||
await assert(actions[1].href.includes('/ops/operations/jobengine/jobs'), 'Jobs secondary action did not target the Job list.', { actions });
|
||||
await assert(!actions.some((entry) => entry.href.includes('/ops/operations/jobs-queues')), 'Jobs tab still contains self-linking row actions.', { actions });
|
||||
checks.push({ name: 'jobs-row-actions', ok: true, actions });
|
||||
|
||||
await searchFilter(page).fill('Vulnerability');
|
||||
await settle(page);
|
||||
let rows = await rowCount(page);
|
||||
await assert(rows === 1, 'Jobs search filter did not reduce the table to one row.', { rows });
|
||||
await page.getByRole('button', { name: 'Clear filters' }).click();
|
||||
await settle(page);
|
||||
rows = await rowCount(page);
|
||||
await assert(rows === 4, 'Jobs clear filters did not restore all rows.', { rows });
|
||||
checks.push({ name: 'jobs-filters', ok: true, rowsAfterClear: rows });
|
||||
|
||||
await searchFilter(page).fill('Vulnerability');
|
||||
await settle(page);
|
||||
await clickTab(page, 'Runs');
|
||||
await assert(await searchFilter(page).inputValue() === '', 'Tab switch did not clear the search filter.');
|
||||
checks.push({ name: 'tab-switch-clears-filters', ok: true });
|
||||
|
||||
await filterSelect(page, 0).selectOption('FAILED');
|
||||
await filterSelect(page, 1).selectOption('DEGRADED');
|
||||
await settle(page);
|
||||
rows = await rowCount(page);
|
||||
await assert(rows === 1, 'Runs filters did not isolate the failed degraded row.', { rows });
|
||||
checks.push({ name: 'runs-filters', ok: true, rows });
|
||||
|
||||
await page.getByRole('button', { name: 'Clear filters' }).click();
|
||||
await settle(page);
|
||||
await page.getByRole('button', { name: 'Copy CorrID' }).first().click();
|
||||
await settle(page);
|
||||
const statusText = ((await actionBanner(page).textContent()) || '').trim();
|
||||
await assert(statusText.includes('corr-run-001'), 'Runs copy action did not produce inline feedback.', { statusText });
|
||||
checks.push({ name: 'runs-copy-feedback', ok: true, statusText });
|
||||
|
||||
await filterSelect(page, 0).selectOption('DEAD-LETTER');
|
||||
await filterSelect(page, 1).selectOption('BLOCKING');
|
||||
await settle(page);
|
||||
await page.getByRole('link', { name: 'Open Dead-Letter Queue' }).click();
|
||||
await settle(page);
|
||||
await assert(page.url().includes('/ops/operations/dead-letter/queue'), 'Runs dead-letter handoff did not reach the DLQ.');
|
||||
checks.push({ name: 'runs-dead-letter-handoff', ok: true, targetUrl: page.url() });
|
||||
|
||||
await gotoPage(page);
|
||||
await clickTab(page, 'Schedules');
|
||||
await filterSelect(page, 0).selectOption('FAIL');
|
||||
await filterSelect(page, 1).selectOption('Daily');
|
||||
await settle(page);
|
||||
rows = await rowCount(page);
|
||||
await assert(rows === 1, 'Schedules filters did not isolate the failed daily schedule.', { rows });
|
||||
await page.getByRole('link', { name: 'Review Dead-Letter Queue' }).click();
|
||||
await settle(page);
|
||||
await assert(page.url().includes('/ops/operations/dead-letter/queue'), 'Schedules failure handoff did not reach the DLQ.');
|
||||
checks.push({ name: 'schedules-failure-handoff', ok: true, targetUrl: page.url() });
|
||||
|
||||
await gotoPage(page);
|
||||
await clickTab(page, 'Schedules');
|
||||
await page.getByRole('link', { name: 'Manage Schedules' }).first().click();
|
||||
await settle(page);
|
||||
await assert(page.url().includes('/ops/operations/scheduler/schedules'), 'Schedules primary handoff did not reach Scheduler.');
|
||||
checks.push({ name: 'schedules-primary-handoff', ok: true, targetUrl: page.url() });
|
||||
|
||||
await gotoPage(page);
|
||||
await clickTab(page, 'Dead Letters');
|
||||
actions = await readActionHrefs(page);
|
||||
await assert(actions[0].href.includes('/ops/operations/dead-letter/queue'), 'Dead-letter primary action did not target the queue.', { actions });
|
||||
await assert(actions[1].href.includes('/ops/operations/data-integrity/dlq'), 'Dead-letter recovery action did not target the DLQ recovery surface.', { actions });
|
||||
await page.getByRole('link', { name: 'Open Replay Recovery' }).first().click();
|
||||
await settle(page);
|
||||
await assert(page.url().includes('/ops/operations/data-integrity/dlq'), 'Dead-letter recovery handoff did not reach Data Integrity.');
|
||||
checks.push({ name: 'dead-letter-handoffs', ok: true, targetUrl: page.url() });
|
||||
|
||||
await gotoPage(page);
|
||||
await clickTab(page, 'Workers');
|
||||
await filterSelect(page, 0).selectOption('DEGRADED');
|
||||
await filterSelect(page, 1).selectOption('feeds');
|
||||
await settle(page);
|
||||
rows = await rowCount(page);
|
||||
await assert(rows === 1, 'Workers filters did not isolate the degraded feeds worker.', { rows });
|
||||
await page.getByRole('link', { name: 'Open Worker Fleet' }).click();
|
||||
await settle(page);
|
||||
await assert(page.url().includes('/ops/operations/scheduler/workers'), 'Workers primary handoff did not reach Worker Fleet.');
|
||||
checks.push({ name: 'workers-primary-handoff', ok: true, targetUrl: page.url() });
|
||||
|
||||
await gotoPage(page);
|
||||
await clickTab(page, 'Workers');
|
||||
await filterSelect(page, 0).selectOption('DEGRADED');
|
||||
await filterSelect(page, 1).selectOption('feeds');
|
||||
await settle(page);
|
||||
await page.getByRole('link', { name: 'Inspect Scheduler Runs' }).click();
|
||||
await settle(page);
|
||||
await assert(page.url().includes('/ops/operations/scheduler/runs'), 'Workers secondary handoff did not reach Scheduler Runs.');
|
||||
checks.push({ name: 'workers-secondary-handoff', ok: true, targetUrl: page.url() });
|
||||
|
||||
const runtimeIssues = {
|
||||
consoleErrors,
|
||||
responseErrors,
|
||||
requestFailures,
|
||||
alerts: await captureRuntimeIssues(page),
|
||||
};
|
||||
const summary = {
|
||||
checkedAtUtc: new Date().toISOString(),
|
||||
routePath,
|
||||
checks,
|
||||
runtimeIssues,
|
||||
failedCheckCount: checks.filter((check) => !check.ok).length,
|
||||
runtimeIssueCount:
|
||||
runtimeIssues.consoleErrors.length
|
||||
+ runtimeIssues.responseErrors.length
|
||||
+ runtimeIssues.requestFailures.length
|
||||
+ runtimeIssues.alerts.length,
|
||||
};
|
||||
|
||||
await writeFile(outputPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
|
||||
|
||||
if (summary.failedCheckCount > 0 || summary.runtimeIssueCount > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
} catch (error) {
|
||||
const summary = {
|
||||
checkedAtUtc: new Date().toISOString(),
|
||||
routePath,
|
||||
checks,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
consoleErrors,
|
||||
responseErrors,
|
||||
requestFailures,
|
||||
};
|
||||
await writeFile(outputPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await context.close().catch(() => {});
|
||||
await browser.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
await run();
|
||||
Reference in New Issue
Block a user