|
|
|
|
@@ -0,0 +1,443 @@
|
|
|
|
|
import fs from 'node:fs';
|
|
|
|
|
import path from 'node:path';
|
|
|
|
|
|
|
|
|
|
import { test, expect } from '../fixtures/auth.fixture';
|
|
|
|
|
import { assertPageHasContent, navigateAndWait } from '../helpers/nav.helper';
|
|
|
|
|
|
|
|
|
|
type RouteTarget = {
|
|
|
|
|
path: string;
|
|
|
|
|
name: string;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
type ControlMeta = {
|
|
|
|
|
tag: string;
|
|
|
|
|
type: string;
|
|
|
|
|
role: string;
|
|
|
|
|
label: string;
|
|
|
|
|
ariaLabel: string;
|
|
|
|
|
disabled: boolean;
|
|
|
|
|
visible: boolean;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
type RouteSweepResult = {
|
|
|
|
|
route: string;
|
|
|
|
|
name: string;
|
|
|
|
|
candidates: number;
|
|
|
|
|
clicked: number;
|
|
|
|
|
skipped: number;
|
|
|
|
|
reasons: SweepReason[];
|
|
|
|
|
skippedByReason: Record<SkipReason, number>;
|
|
|
|
|
failures: string[];
|
|
|
|
|
angularErrors: string[];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
type SweepReason = 'no-controls' | 'guarded' | 'occluded' | 'error-state' | 'clicked';
|
|
|
|
|
type SkipReason = 'hidden' | 'disabled' | 'submit' | 'destructive' | 'unlabeled' | 'unsupported' | 'occluded';
|
|
|
|
|
|
|
|
|
|
const RAW_ROUTES: RouteTarget[] = [
|
|
|
|
|
{ path: '/', name: 'Control Plane' },
|
|
|
|
|
{ path: '/approvals', name: 'Approvals' },
|
|
|
|
|
{ path: '/releases', name: 'Releases' },
|
|
|
|
|
{ path: '/deployments', name: 'Deployments' },
|
|
|
|
|
{ path: '/security', name: 'Security Overview' },
|
|
|
|
|
{ path: '/security/overview', name: 'Security Overview Detail' },
|
|
|
|
|
{ path: '/security/findings', name: 'Security Findings' },
|
|
|
|
|
{ path: '/security/vulnerabilities', name: 'Security Vulnerabilities' },
|
|
|
|
|
{ path: '/security/vex', name: 'Security VEX' },
|
|
|
|
|
{ path: '/policy', name: 'Policy' },
|
|
|
|
|
{ path: '/policy/packs', name: 'Policy Packs' },
|
|
|
|
|
{ path: '/policy/governance', name: 'Policy Governance' },
|
|
|
|
|
{ path: '/policy/exceptions', name: 'Policy Exceptions' },
|
|
|
|
|
{ path: '/operations', name: 'Operations' },
|
|
|
|
|
{ path: '/operations/orchestrator', name: 'Operations Orchestrator' },
|
|
|
|
|
{ path: '/operations/scheduler', name: 'Operations Scheduler' },
|
|
|
|
|
{ path: '/evidence', name: 'Evidence' },
|
|
|
|
|
{ path: '/evidence-packs', name: 'Evidence Packs' },
|
|
|
|
|
{ path: '/settings', name: 'Settings' },
|
|
|
|
|
{ path: '/console/profile', name: 'Profile' },
|
|
|
|
|
{ path: '/admin/trust', name: 'Trust Admin' },
|
|
|
|
|
{ path: '/admin/vex-hub', name: 'VEX Hub Admin' },
|
|
|
|
|
{ path: '/integrations', name: 'Integration Hub' },
|
|
|
|
|
{ path: '/findings', name: 'Findings' },
|
|
|
|
|
{ path: '/triage', name: 'Triage Canvas' },
|
|
|
|
|
{ path: '/environments', name: 'Environments' },
|
|
|
|
|
{ path: '/home', name: 'Home Dashboard Legacy' },
|
|
|
|
|
{ path: '/dashboard/sources', name: 'Sources Dashboard' },
|
|
|
|
|
{ path: '/console/status', name: 'Console Status' },
|
|
|
|
|
{ path: '/console/admin', name: 'Console Admin' },
|
|
|
|
|
{ path: '/console/configuration', name: 'Configuration' },
|
|
|
|
|
{ path: '/orchestrator', name: 'Orchestrator Legacy' },
|
|
|
|
|
{ path: '/orchestrator/jobs', name: 'Orchestrator Jobs' },
|
|
|
|
|
{ path: '/orchestrator/quotas', name: 'Orchestrator Quotas' },
|
|
|
|
|
{ path: '/release-orchestrator', name: 'Release Orchestrator' },
|
|
|
|
|
{ path: '/policy-studio/packs', name: 'Policy Studio Packs' },
|
|
|
|
|
{ path: '/concelier/trivy-db-settings', name: 'Trivy DB Settings' },
|
|
|
|
|
{ path: '/risk', name: 'Risk Dashboard' },
|
|
|
|
|
{ path: '/graph', name: 'Graph Explorer' },
|
|
|
|
|
{ path: '/lineage', name: 'Lineage' },
|
|
|
|
|
{ path: '/reachability', name: 'Reachability Center' },
|
|
|
|
|
{ path: '/timeline', name: 'Timeline' },
|
|
|
|
|
{ path: '/evidence-thread', name: 'Evidence Thread' },
|
|
|
|
|
{ path: '/vulnerabilities', name: 'Vulnerability Explorer' },
|
|
|
|
|
{ path: '/vulnerabilities/triage', name: 'Vulnerability Triage' },
|
|
|
|
|
{ path: '/triage/inbox', name: 'Triage Inbox' },
|
|
|
|
|
{ path: '/triage/artifacts', name: 'Triage Artifacts' },
|
|
|
|
|
{ path: '/triage/quiet-lane', name: 'Quiet Lane' },
|
|
|
|
|
{ path: '/triage/ai-recommendations', name: 'AI Recommendations' },
|
|
|
|
|
{ path: '/notify', name: 'Notify Panel' },
|
|
|
|
|
{ path: '/admin/notifications', name: 'Admin Notifications' },
|
|
|
|
|
{ path: '/ops/feeds', name: 'Feed Mirror' },
|
|
|
|
|
{ path: '/ops/signals', name: 'Signals Dashboard' },
|
|
|
|
|
{ path: '/ops/packs', name: 'Pack Registry Browser' },
|
|
|
|
|
{ path: '/admin/policy/governance', name: 'Policy Governance Admin' },
|
|
|
|
|
{ path: '/admin/policy/simulation', name: 'Policy Simulation Admin' },
|
|
|
|
|
{ path: '/scheduler', name: 'Scheduler' },
|
|
|
|
|
{ path: '/exceptions', name: 'Exceptions' },
|
|
|
|
|
{ path: '/admin/registries', name: 'Registry Admin' },
|
|
|
|
|
{ path: '/admin/issuers', name: 'Issuer Trust' },
|
|
|
|
|
{ path: '/ops/scanner', name: 'Scanner Ops' },
|
|
|
|
|
{ path: '/ops/offline-kit', name: 'Offline Kit' },
|
|
|
|
|
{ path: '/ops/aoc', name: 'AOC Compliance' },
|
|
|
|
|
{ path: '/admin/audit', name: 'Audit Log' },
|
|
|
|
|
{ path: '/welcome', name: 'Welcome Page' },
|
|
|
|
|
{ path: '/ops/quotas', name: 'Quota Dashboard' },
|
|
|
|
|
{ path: '/ops/orchestrator/dead-letter', name: 'Dead Letter Queue' },
|
|
|
|
|
{ path: '/ops/orchestrator/slo', name: 'SLO Burn Rate' },
|
|
|
|
|
{ path: '/ops/health', name: 'Platform Health' },
|
|
|
|
|
{ path: '/ops/doctor', name: 'Doctor Diagnostics' },
|
|
|
|
|
{ path: '/ops/agents', name: 'Agent Fleet' },
|
|
|
|
|
{ path: '/analyze/unknowns', name: 'Unknowns Tracking' },
|
|
|
|
|
{ path: '/analyze/patch-map', name: 'Patch Map Explorer' },
|
|
|
|
|
{ path: '/ops/binary-index', name: 'Binary Index Ops' },
|
|
|
|
|
{ path: '/settings/determinization-config', name: 'Determinization Config' },
|
|
|
|
|
{ path: '/sbom-sources', name: 'SBOM Sources' },
|
|
|
|
|
{ path: '/sbom/diff', name: 'SBOM Diff' },
|
|
|
|
|
{ path: '/deploy/diff', name: 'Deploy Diff' },
|
|
|
|
|
{ path: '/vex/timeline', name: 'VEX Timeline' },
|
|
|
|
|
{ path: '/workspace/dev', name: 'Developer Workspace' },
|
|
|
|
|
{ path: '/workspace/audit', name: 'Auditor Workspace' },
|
|
|
|
|
{ path: '/ai/autofix', name: 'AI Autofix' },
|
|
|
|
|
{ path: '/ai/chat', name: 'AI Chat' },
|
|
|
|
|
{ path: '/ai/chips', name: 'AI Chips' },
|
|
|
|
|
{ path: '/ai-runs', name: 'AI Runs' },
|
|
|
|
|
{ path: '/change-trace', name: 'Change Trace' },
|
|
|
|
|
{ path: '/aoc/verify', name: 'AOC Verification' },
|
|
|
|
|
{ path: '/audit/reasons', name: 'Audit Reasons' },
|
|
|
|
|
{ path: '/triage/audit-bundles', name: 'Triage Audit Bundles' },
|
|
|
|
|
{ path: '/setup', name: 'Setup' },
|
|
|
|
|
{ path: '/setup/wizard', name: 'Setup Wizard' },
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
const ROUTES: RouteTarget[] = RAW_ROUTES.filter(
|
|
|
|
|
(route, index, all) => all.findIndex((candidate) => candidate.path === route.path) === index
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
const DESTRUCTIVE_LABEL_PATTERN = /\b(delete|remove|purge|destroy|drop|wipe|revoke|terminate|logout|sign out|clear all)\b/i;
|
|
|
|
|
|
|
|
|
|
const KNOWN_NO_CONTROL_ROUTES = new Set<string>([
|
|
|
|
|
'/security/vulnerabilities',
|
|
|
|
|
'/policy/governance',
|
|
|
|
|
'/evidence',
|
|
|
|
|
'/console/profile',
|
|
|
|
|
'/triage/inbox',
|
|
|
|
|
'/operations',
|
|
|
|
|
'/operations/orchestrator',
|
|
|
|
|
'/orchestrator',
|
|
|
|
|
'/orchestrator/jobs',
|
|
|
|
|
'/orchestrator/quotas',
|
|
|
|
|
'/admin/audit',
|
|
|
|
|
'/workspace/dev',
|
|
|
|
|
'/workspace/audit',
|
|
|
|
|
'/sbom/diff',
|
|
|
|
|
'/deploy/diff',
|
|
|
|
|
'/vex/timeline',
|
|
|
|
|
'/admin/registries',
|
|
|
|
|
'/admin/issuers',
|
|
|
|
|
'/admin/policy/governance',
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
const SWEEP_THRESHOLDS = {
|
|
|
|
|
minRouteCoverage: 0.75,
|
|
|
|
|
minActionCoverage: 0.40,
|
|
|
|
|
maxSkipRatio: 0.75,
|
|
|
|
|
maxErrorStateRoutes: 0,
|
|
|
|
|
maxUnreviewedNoControlRoutes: 0,
|
|
|
|
|
} as const;
|
|
|
|
|
|
|
|
|
|
const GUARDED_SKIP_REASONS: SkipReason[] = ['hidden', 'disabled', 'submit', 'destructive', 'unlabeled', 'unsupported'];
|
|
|
|
|
|
|
|
|
|
function addReason(result: RouteSweepResult, reason: SweepReason): void {
|
|
|
|
|
if (!result.reasons.includes(reason)) {
|
|
|
|
|
result.reasons.push(reason);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function describeControl(handle: import('@playwright/test').ElementHandle<SVGElement | HTMLElement>): Promise<ControlMeta | null> {
|
|
|
|
|
return handle.evaluate((node) => {
|
|
|
|
|
const element = node as HTMLElement;
|
|
|
|
|
const rect = element.getBoundingClientRect();
|
|
|
|
|
const style = window.getComputedStyle(element);
|
|
|
|
|
const tag = element.tagName.toLowerCase();
|
|
|
|
|
const type = (element.getAttribute('type') ?? '').toLowerCase();
|
|
|
|
|
const role = (element.getAttribute('role') ?? '').toLowerCase();
|
|
|
|
|
const label = (element.innerText ?? element.textContent ?? '').replace(/\s+/g, ' ').trim();
|
|
|
|
|
const ariaLabel = (element.getAttribute('aria-label') ?? element.getAttribute('title') ?? '').replace(/\s+/g, ' ').trim();
|
|
|
|
|
const disabled =
|
|
|
|
|
element.hasAttribute('disabled') ||
|
|
|
|
|
element.getAttribute('aria-disabled') === 'true' ||
|
|
|
|
|
(element as HTMLButtonElement).disabled === true;
|
|
|
|
|
const visible =
|
|
|
|
|
rect.width > 0 &&
|
|
|
|
|
rect.height > 0 &&
|
|
|
|
|
style.display !== 'none' &&
|
|
|
|
|
style.visibility !== 'hidden' &&
|
|
|
|
|
style.opacity !== '0';
|
|
|
|
|
|
|
|
|
|
if (tag !== 'button' && role !== 'button') {
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
tag,
|
|
|
|
|
type,
|
|
|
|
|
role,
|
|
|
|
|
label,
|
|
|
|
|
ariaLabel,
|
|
|
|
|
disabled,
|
|
|
|
|
visible,
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
test.describe('Workflow: Exhaustive Button Sweep', () => {
|
|
|
|
|
test('clicks visible non-destructive controls on all routes', async ({ authenticatedPage: page }, testInfo) => {
|
|
|
|
|
test.setTimeout(45 * 60 * 1000);
|
|
|
|
|
|
|
|
|
|
const consoleErrors: string[] = [];
|
|
|
|
|
const pageErrors: string[] = [];
|
|
|
|
|
const results: RouteSweepResult[] = [];
|
|
|
|
|
|
|
|
|
|
page.on('console', (message) => {
|
|
|
|
|
if (message.type() === 'error') {
|
|
|
|
|
consoleErrors.push(message.text());
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
page.on('pageerror', (error) => {
|
|
|
|
|
pageErrors.push(error.message);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
page.on('dialog', (dialog) => {
|
|
|
|
|
void dialog.dismiss();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
for (const route of ROUTES) {
|
|
|
|
|
const routeResult: RouteSweepResult = {
|
|
|
|
|
route: route.path,
|
|
|
|
|
name: route.name,
|
|
|
|
|
candidates: 0,
|
|
|
|
|
clicked: 0,
|
|
|
|
|
skipped: 0,
|
|
|
|
|
reasons: [],
|
|
|
|
|
skippedByReason: {
|
|
|
|
|
hidden: 0,
|
|
|
|
|
disabled: 0,
|
|
|
|
|
submit: 0,
|
|
|
|
|
destructive: 0,
|
|
|
|
|
unlabeled: 0,
|
|
|
|
|
unsupported: 0,
|
|
|
|
|
occluded: 0,
|
|
|
|
|
},
|
|
|
|
|
failures: [],
|
|
|
|
|
angularErrors: [],
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const consoleStart = consoleErrors.length;
|
|
|
|
|
const pageErrorStart = pageErrors.length;
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
await navigateAndWait(page, route.path, { timeout: 40_000 });
|
|
|
|
|
await page.waitForTimeout(1200);
|
|
|
|
|
await assertPageHasContent(page);
|
|
|
|
|
} catch (error) {
|
|
|
|
|
routeResult.failures.push(`route_load_failed: ${String(error)}`);
|
|
|
|
|
addReason(routeResult, 'error-state');
|
|
|
|
|
results.push(routeResult);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const root = (await page.locator('#main-content').count()) > 0 ? '#main-content' : 'body';
|
|
|
|
|
const controls = await page.locator(`${root} button, ${root} [role="button"]`).elementHandles();
|
|
|
|
|
routeResult.candidates = controls.length;
|
|
|
|
|
if (controls.length === 0) {
|
|
|
|
|
addReason(routeResult, 'no-controls');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const control of controls) {
|
|
|
|
|
let meta: ControlMeta | null = null;
|
|
|
|
|
try {
|
|
|
|
|
meta = await describeControl(control);
|
|
|
|
|
} catch {
|
|
|
|
|
routeResult.skipped += 1;
|
|
|
|
|
routeResult.skippedByReason.unsupported += 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!meta) {
|
|
|
|
|
routeResult.skipped += 1;
|
|
|
|
|
routeResult.skippedByReason.unsupported += 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const label = `${meta.label} ${meta.ariaLabel}`.trim();
|
|
|
|
|
let skipReason: SkipReason | null = null;
|
|
|
|
|
if (!meta.visible) {
|
|
|
|
|
skipReason = 'hidden';
|
|
|
|
|
} else if (meta.disabled) {
|
|
|
|
|
skipReason = 'disabled';
|
|
|
|
|
} else if (meta.type === 'submit') {
|
|
|
|
|
skipReason = 'submit';
|
|
|
|
|
} else if (!label) {
|
|
|
|
|
skipReason = 'unlabeled';
|
|
|
|
|
} else if (DESTRUCTIVE_LABEL_PATTERN.test(label)) {
|
|
|
|
|
skipReason = 'destructive';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (skipReason) {
|
|
|
|
|
routeResult.skipped += 1;
|
|
|
|
|
routeResult.skippedByReason[skipReason] += 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
await control.scrollIntoViewIfNeeded();
|
|
|
|
|
await control.click({ timeout: 3_000 });
|
|
|
|
|
routeResult.clicked += 1;
|
|
|
|
|
await page.waitForTimeout(250);
|
|
|
|
|
|
|
|
|
|
if (!page.url().includes(route.path)) {
|
|
|
|
|
await navigateAndWait(page, route.path, { timeout: 30_000 });
|
|
|
|
|
await page.waitForTimeout(500);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await page.keyboard.press('Escape').catch(() => undefined);
|
|
|
|
|
} catch (error) {
|
|
|
|
|
const errorText = String(error);
|
|
|
|
|
if (errorText.includes('intercepts pointer events')) {
|
|
|
|
|
routeResult.skipped += 1;
|
|
|
|
|
routeResult.skippedByReason.occluded += 1;
|
|
|
|
|
await page.keyboard.press('Escape').catch(() => undefined);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
routeResult.failures.push(`click_failed "${label}": ${errorText}`);
|
|
|
|
|
await page.keyboard.press('Escape').catch(() => undefined);
|
|
|
|
|
if (!page.url().includes(route.path)) {
|
|
|
|
|
await navigateAndWait(page, route.path, { timeout: 30_000 }).catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const routeConsoleErrors = consoleErrors.slice(consoleStart);
|
|
|
|
|
const routePageErrors = pageErrors.slice(pageErrorStart);
|
|
|
|
|
routeResult.angularErrors = [...routeConsoleErrors, ...routePageErrors].filter((error) =>
|
|
|
|
|
/NG0\d{3,4}/.test(error)
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
if (routeResult.clicked > 0) {
|
|
|
|
|
addReason(routeResult, 'clicked');
|
|
|
|
|
}
|
|
|
|
|
if (routeResult.skippedByReason.occluded > 0) {
|
|
|
|
|
addReason(routeResult, 'occluded');
|
|
|
|
|
}
|
|
|
|
|
const guardedSkips = GUARDED_SKIP_REASONS.reduce((sum, key) => sum + routeResult.skippedByReason[key], 0);
|
|
|
|
|
if (guardedSkips > 0) {
|
|
|
|
|
addReason(routeResult, 'guarded');
|
|
|
|
|
}
|
|
|
|
|
if (routeResult.angularErrors.length > 0 || routeResult.failures.length > 0) {
|
|
|
|
|
addReason(routeResult, 'error-state');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
results.push(routeResult);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const totalCandidates = results.reduce((sum, item) => sum + item.candidates, 0);
|
|
|
|
|
const totalClicked = results.reduce((sum, item) => sum + item.clicked, 0);
|
|
|
|
|
const totalSkipped = results.reduce((sum, item) => sum + item.skipped, 0);
|
|
|
|
|
const totalFailures = results.reduce((sum, item) => sum + item.failures.length, 0);
|
|
|
|
|
const actionableRoutes = results.filter((item) => item.candidates > 0).length;
|
|
|
|
|
const clickedRoutes = results.filter((item) => item.clicked > 0).length;
|
|
|
|
|
const routeCoverage = actionableRoutes > 0 ? clickedRoutes / actionableRoutes : 1;
|
|
|
|
|
const actionCoverage = totalCandidates > 0 ? totalClicked / totalCandidates : 1;
|
|
|
|
|
const skipRatio = totalCandidates > 0 ? totalSkipped / totalCandidates : 0;
|
|
|
|
|
const zeroCandidateRoutes = results.filter((item) => item.candidates === 0);
|
|
|
|
|
const reviewedNoControlRoutes = zeroCandidateRoutes
|
|
|
|
|
.filter((item) => KNOWN_NO_CONTROL_ROUTES.has(item.route))
|
|
|
|
|
.map((item) => item.route);
|
|
|
|
|
const unreviewedNoControlRoutes = zeroCandidateRoutes
|
|
|
|
|
.filter((item) => !KNOWN_NO_CONTROL_ROUTES.has(item.route))
|
|
|
|
|
.map((item) => item.route);
|
|
|
|
|
const errorStateRoutes = results
|
|
|
|
|
.filter((item) => item.reasons.includes('error-state'))
|
|
|
|
|
.map((item) => item.route);
|
|
|
|
|
const failedChecks: string[] = [];
|
|
|
|
|
|
|
|
|
|
if (routeCoverage < SWEEP_THRESHOLDS.minRouteCoverage) {
|
|
|
|
|
failedChecks.push(
|
|
|
|
|
`routeCoverage ${routeCoverage.toFixed(3)} below threshold ${SWEEP_THRESHOLDS.minRouteCoverage.toFixed(3)}`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (actionCoverage < SWEEP_THRESHOLDS.minActionCoverage) {
|
|
|
|
|
failedChecks.push(
|
|
|
|
|
`actionCoverage ${actionCoverage.toFixed(3)} below threshold ${SWEEP_THRESHOLDS.minActionCoverage.toFixed(3)}`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (skipRatio > SWEEP_THRESHOLDS.maxSkipRatio) {
|
|
|
|
|
failedChecks.push(`skipRatio ${skipRatio.toFixed(3)} above threshold ${SWEEP_THRESHOLDS.maxSkipRatio.toFixed(3)}`);
|
|
|
|
|
}
|
|
|
|
|
if (errorStateRoutes.length > SWEEP_THRESHOLDS.maxErrorStateRoutes) {
|
|
|
|
|
failedChecks.push(
|
|
|
|
|
`errorStateRoutes ${errorStateRoutes.length} exceeds threshold ${SWEEP_THRESHOLDS.maxErrorStateRoutes}: ${errorStateRoutes.join(
|
|
|
|
|
', '
|
|
|
|
|
)}`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (unreviewedNoControlRoutes.length > SWEEP_THRESHOLDS.maxUnreviewedNoControlRoutes) {
|
|
|
|
|
failedChecks.push(
|
|
|
|
|
`unreviewedNoControlRoutes ${unreviewedNoControlRoutes.length} exceeds threshold ${SWEEP_THRESHOLDS.maxUnreviewedNoControlRoutes}: ${unreviewedNoControlRoutes.join(
|
|
|
|
|
', '
|
|
|
|
|
)}`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const report = {
|
|
|
|
|
generatedAtUtc: new Date().toISOString(),
|
|
|
|
|
totalRoutes: results.length,
|
|
|
|
|
totalCandidates,
|
|
|
|
|
totalClicked,
|
|
|
|
|
totalSkipped,
|
|
|
|
|
totalFailures,
|
|
|
|
|
coverage: {
|
|
|
|
|
actionableRoutes,
|
|
|
|
|
clickedRoutes,
|
|
|
|
|
routeCoverage,
|
|
|
|
|
actionCoverage,
|
|
|
|
|
skipRatio,
|
|
|
|
|
zeroCandidateRoutes: zeroCandidateRoutes.length,
|
|
|
|
|
reviewedNoControlRoutes,
|
|
|
|
|
unreviewedNoControlRoutes,
|
|
|
|
|
errorStateRoutes,
|
|
|
|
|
thresholds: SWEEP_THRESHOLDS,
|
|
|
|
|
failedChecks,
|
|
|
|
|
},
|
|
|
|
|
routes: results,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const outputDirectory = path.resolve(process.cwd(), 'test-results');
|
|
|
|
|
fs.mkdirSync(outputDirectory, { recursive: true });
|
|
|
|
|
const outputFile = path.join(outputDirectory, 'exhaustive-button-sweep-report.json');
|
|
|
|
|
fs.writeFileSync(outputFile, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
|
|
|
await testInfo.attach('exhaustive-button-sweep-report', { path: outputFile, contentType: 'application/json' });
|
|
|
|
|
expect(failedChecks, `Coverage gates failed. See ${outputFile}`).toEqual([]);
|
|
|
|
|
});
|
|
|
|
|
});
|