Close admin trust audit gaps and stabilize live sweeps

This commit is contained in:
master
2026-03-12 10:14:00 +02:00
parent a00efb7ab2
commit 6964a046a5
50 changed files with 5968 additions and 2850 deletions

View File

@@ -0,0 +1,252 @@
#!/usr/bin/env node
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const webRoot = path.resolve(__dirname, '..');
const outputDir = path.join(webRoot, 'output', 'playwright');
const resultPath = path.join(outputDir, 'live-full-core-audit.json');
const suites = [
{
name: 'frontdoor-canonical-route-sweep',
script: 'live-frontdoor-canonical-route-sweep.mjs',
reportPath: path.join(outputDir, 'live-frontdoor-canonical-route-sweep.json'),
},
{
name: 'mission-control-action-sweep',
script: 'live-mission-control-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-mission-control-action-sweep.json'),
},
{
name: 'ops-policy-action-sweep',
script: 'live-ops-policy-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-ops-policy-action-sweep.json'),
},
{
name: 'integrations-action-sweep',
script: 'live-integrations-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-integrations-action-sweep.json'),
},
{
name: 'setup-topology-action-sweep',
script: 'live-setup-topology-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-setup-topology-action-sweep.json'),
},
{
name: 'setup-admin-action-sweep',
script: 'live-setup-admin-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-setup-admin-action-sweep.json'),
},
{
name: 'user-reported-admin-trust-check',
script: 'live-user-reported-admin-trust-check.mjs',
reportPath: path.join(outputDir, 'live-user-reported-admin-trust-check.json'),
},
{
name: 'jobs-queues-action-sweep',
script: 'live-jobs-queues-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-jobs-queues-action-sweep.json'),
},
{
name: 'triage-artifacts-scope-compat',
script: 'live-triage-artifacts-scope-compat.mjs',
reportPath: path.join(outputDir, 'live-triage-artifacts-scope-compat.json'),
},
{
name: 'releases-deployments-check',
script: 'live-releases-deployments-check.mjs',
reportPath: path.join(outputDir, 'live-releases-deployments-check.json'),
},
{
name: 'release-promotion-submit-check',
script: 'live-release-promotion-submit-check.mjs',
reportPath: path.join(outputDir, 'live-release-promotion-submit-check.json'),
},
{
name: 'hotfix-action-check',
script: 'live-hotfix-action-check.mjs',
reportPath: path.join(outputDir, 'live-hotfix-action-check.json'),
},
{
name: 'registry-admin-audit-check',
script: 'live-registry-admin-audit-check.mjs',
reportPath: path.join(outputDir, 'live-registry-admin-audit-check.json'),
},
{
name: 'evidence-export-action-sweep',
script: 'live-evidence-export-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-evidence-export-action-sweep.json'),
},
{
name: 'watchlist-action-sweep',
script: 'live-watchlist-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-watchlist-action-sweep.json'),
},
{
name: 'notifications-watchlist-recheck',
script: 'live-notifications-watchlist-recheck.mjs',
reportPath: path.join(outputDir, 'live-notifications-watchlist-recheck.json'),
},
{
name: 'policy-simulation-direct-routes',
script: 'live-policy-simulation-direct-routes.mjs',
reportPath: path.join(outputDir, 'live-policy-simulation-direct-routes.json'),
},
{
name: 'uncovered-surface-action-sweep',
script: 'live-uncovered-surface-action-sweep.mjs',
reportPath: path.join(outputDir, 'live-uncovered-surface-action-sweep.json'),
},
{
name: 'unified-search-route-matrix',
script: 'live-frontdoor-unified-search-route-matrix.mjs',
reportPath: path.join(outputDir, 'live-frontdoor-unified-search-route-matrix.json'),
},
];
const failureCountKeys = new Set([
'failedRouteCount',
'failedCheckCount',
'failedChecks',
'failedActionCount',
'failedCount',
'failureCount',
'errorCount',
'runtimeIssueCount',
'issueCount',
'unexpectedErrorCount',
]);
const arrayFailureKeys = new Set([
'failures',
'runtimeIssues',
'runtimeErrors',
'errors',
'warnings',
]);
function collectFailureSignals(value) {
const signals = [];
function visit(node, trail = []) {
if (Array.isArray(node)) {
if (trail.at(-1) && arrayFailureKeys.has(trail.at(-1)) && node.length > 0) {
signals.push({ path: trail.join('.'), count: node.length });
}
const failingEntries = node.filter((entry) => entry && typeof entry === 'object' && entry.ok === false);
if (failingEntries.length > 0) {
signals.push({ path: trail.join('.'), count: failingEntries.length });
}
node.forEach((entry, index) => visit(entry, [...trail, String(index)]));
return;
}
if (!node || typeof node !== 'object') {
return;
}
for (const [key, child] of Object.entries(node)) {
if (typeof child === 'number' && failureCountKeys.has(key) && child > 0) {
signals.push({ path: [...trail, key].join('.'), count: child });
}
visit(child, [...trail, key]);
}
}
visit(value);
return signals;
}
async function readReport(reportPath) {
try {
const content = await readFile(reportPath, 'utf8');
return JSON.parse(content);
} catch (error) {
return {
reportReadFailed: true,
error: error instanceof Error ? error.message : String(error),
};
}
}
function runSuite({ name, script }) {
return new Promise((resolve) => {
const startedAt = Date.now();
const child = spawn(process.execPath, [path.join(__dirname, script)], {
cwd: webRoot,
env: process.env,
stdio: 'inherit',
});
child.on('exit', (code, signal) => {
resolve({
name,
script,
exitCode: code ?? null,
signal: signal ?? null,
durationMs: Date.now() - startedAt,
});
});
});
}
async function main() {
await mkdir(outputDir, { recursive: true });
const summary = {
generatedAtUtc: new Date().toISOString(),
baseUrl: process.env.STELLAOPS_FRONTDOOR_BASE_URL?.trim() || 'https://stella-ops.local',
suiteCount: suites.length,
suites: [],
};
for (const suite of suites) {
process.stdout.write(`[live-full-core-audit] START ${suite.name}\n`);
const execution = await runSuite(suite);
const report = await readReport(suite.reportPath);
const failureSignals = collectFailureSignals(report);
const ok = execution.exitCode === 0 && failureSignals.length === 0 && !report.reportReadFailed;
const result = {
...execution,
reportPath: suite.reportPath,
ok,
failureSignals,
report,
};
summary.suites.push(result);
process.stdout.write(
`[live-full-core-audit] DONE ${suite.name} ok=${ok} exitCode=${execution.exitCode ?? 'null'} ` +
`signals=${failureSignals.length} durationMs=${execution.durationMs}\n`,
);
}
summary.failedSuiteCount = summary.suites.filter((suite) => !suite.ok).length;
summary.passedSuiteCount = summary.suiteCount - summary.failedSuiteCount;
summary.failedSuites = summary.suites
.filter((suite) => !suite.ok)
.map((suite) => ({
name: suite.name,
exitCode: suite.exitCode,
signal: suite.signal,
failureSignals: suite.failureSignals,
reportPath: suite.reportPath,
}));
await writeFile(resultPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
if (summary.failedSuiteCount > 0) {
process.exitCode = 1;
}
}
await main();

View File

@@ -239,6 +239,28 @@ async function waitForAnyButton(page, names, timeoutMs = ELEMENT_WAIT_MS) {
return null;
}
async function waitForEnabledButton(page, names, timeoutMs = ELEMENT_WAIT_MS) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
for (const name of names) {
const locator = page.getByRole('button', { name });
const count = await locator.count();
for (let index = 0; index < count; index += 1) {
const candidate = locator.nth(index);
const disabled = await candidate.isDisabled().catch(() => true);
if (!disabled) {
return { name, locator: candidate };
}
}
}
await page.waitForTimeout(250);
}
return null;
}
async function clickLink(context, page, route, name, index = 0) {
await navigate(page, route);
const target = await waitForNavigationTarget(page, name, index);
@@ -380,8 +402,8 @@ async function exerciseShadowResults(page) {
let restoredDisabledState = false;
if (initiallyDisabled) {
const enableButton = page.getByRole('button', { name: 'Enable' }).first();
if ((await enableButton.count()) === 0) {
const enableTarget = await waitForEnabledButton(page, ['Enable Shadow Mode', /^Enable$/], 12_000);
if (!enableTarget) {
return {
action: 'button:View Results',
ok: false,
@@ -391,16 +413,13 @@ async function exerciseShadowResults(page) {
};
}
await enableButton.click({ timeout: 10_000 });
await enableTarget.locator.click({ timeout: 10_000 });
enabledInFlow = true;
await Promise.race([
page.waitForFunction(() => {
const buttons = Array.from(document.querySelectorAll('button'));
const button = buttons.find((candidate) => (candidate.textContent || '').trim() === 'View Results');
return button instanceof HTMLButtonElement && !button.disabled;
}, null, { timeout: 12_000 }).catch(() => {}),
page.waitForTimeout(2_000),
]);
await page.waitForFunction(() => {
const buttons = Array.from(document.querySelectorAll('button'));
const button = buttons.find((candidate) => (candidate.textContent || '').trim() === 'View Results');
return button instanceof HTMLButtonElement && !button.disabled;
}, null, { timeout: 12_000 }).catch(() => {});
steps.push({
step: 'enable-shadow-mode',
snapshot: await captureSnapshot(page, 'policy-simulation:enabled-shadow-mode'),

View File

@@ -20,7 +20,7 @@ const topologyScope = {
timeWindow: '7d',
};
const topologyScopeQuery = new URLSearchParams(topologyScope).toString();
const STEP_TIMEOUT_MS = 30_000;
const STEP_TIMEOUT_MS = 60_000;
const GENERIC_TITLES = new Set(['StellaOps', 'Stella Ops Dashboard']);
const ROUTE_READINESS = [
{
@@ -41,17 +41,17 @@ const ROUTE_READINESS = [
{
path: '/setup/topology/targets',
title: 'Targets - StellaOps',
markers: ['No targets for current filters.', 'Select a target row to view its topology mapping details.'],
markers: ['Targets', 'Selected Target', 'No targets for current filters.'],
},
{
path: '/setup/topology/hosts',
title: 'Hosts - StellaOps',
markers: ['No hosts for current filters.', 'Select a host row to inspect runtime drift and impact.'],
markers: ['Hosts', 'Selected Host', 'No hosts for current filters.'],
},
{
path: '/setup/topology/agents',
title: 'Agent Fleet - StellaOps',
markers: ['No groups for current filters.', 'All Agents', 'View Targets'],
markers: ['Agent Groups', 'All Agents', 'No groups for current filters.'],
},
];
@@ -135,6 +135,10 @@ function routeReadiness(pathname) {
return ROUTE_READINESS.find((entry) => entry.path === pathname) ?? null;
}
function normalizeReadyTitle(value) {
return value.toLowerCase().replace(/[^a-z0-9]+/g, '');
}
async function waitForRouteReady(page, routeOrPath) {
const expectedPath = routePath(routeOrPath);
const readiness = routeReadiness(expectedPath);
@@ -165,7 +169,12 @@ async function waitForRouteReady(page, routeOrPath) {
}
const title = document.title.trim();
if (title.length === 0 || genericTitles.includes(title) || title !== expectedTitle) {
const normalizedTitle = title.toLowerCase().replace(/[^a-z0-9]+/g, '');
if (
title.length === 0 ||
genericTitles.includes(title) ||
!normalizedTitle.includes(expectedTitle)
) {
return false;
}
@@ -174,7 +183,7 @@ async function waitForRouteReady(page, routeOrPath) {
},
{
expectedPathValue: expectedPath,
expectedTitle: readiness.title,
expectedTitle: normalizeReadyTitle(readiness.title),
markers: readiness.markers,
genericTitles: [...GENERIC_TITLES],
},

View File

@@ -0,0 +1,423 @@
#!/usr/bin/env node
import { mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { chromium } from 'playwright';
import { authenticateFrontdoor, createAuthenticatedContext } from './live-frontdoor-auth.mjs';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const webRoot = path.resolve(__dirname, '..');
const outputDir = path.join(webRoot, 'output', 'playwright');
const outputPath = path.join(outputDir, 'live-uncovered-surface-action-sweep.json');
const authStatePath = path.join(outputDir, 'live-uncovered-surface-action-sweep.state.json');
const authReportPath = path.join(outputDir, 'live-uncovered-surface-action-sweep.auth.json');
const baseUrl = process.env.STELLAOPS_FRONTDOOR_BASE_URL?.trim() || 'https://stella-ops.local';
const scopeQuery = 'tenant=demo-prod&regions=us-east&environments=stage&timeWindow=7d';
const linkChecks = [
['/releases/overview', 'Release Versions', '/releases/versions'],
['/releases/overview', 'Release Runs', '/releases/runs'],
['/releases/overview', 'Approvals Queue', '/releases/approvals'],
['/releases/overview', 'Hotfixes', '/releases/hotfixes'],
['/releases/overview', 'Promotions', '/releases/promotions'],
['/releases/overview', 'Deployment History', '/releases/deployments'],
['/releases/runs', 'Timeline', '/releases/runs?view=timeline'],
['/releases/runs', 'Table', '/releases/runs?view=table'],
['/releases/runs', 'Correlations', '/releases/runs?view=correlations'],
['/releases/approvals', 'Pending', '/releases/approvals?tab=pending'],
['/releases/approvals', 'Approved', '/releases/approvals?tab=approved'],
['/releases/approvals', 'Rejected', '/releases/approvals?tab=rejected'],
['/releases/approvals', 'Expiring', '/releases/approvals?tab=expiring'],
['/releases/approvals', 'My Team', '/releases/approvals?tab=my-team'],
['/releases/environments', 'Open Environment', '/setup/topology/environments/stage/posture'],
['/releases/environments', 'Open Targets', '/setup/topology/targets'],
['/releases/environments', 'Open Agents', '/setup/topology/agents'],
['/releases/environments', 'Open Runs', '/releases/runs'],
['/releases/investigation/deploy-diff', 'Open Deployments', '/releases/deployments'],
['/releases/investigation/deploy-diff', 'Open Releases Overview', '/releases/overview'],
['/releases/investigation/change-trace', 'Open Deployments', '/releases/deployments'],
['/security/posture', 'Open triage', '/security/triage'],
['/security/posture', 'Disposition', '/security/disposition'],
['/security/posture', 'Configure sources', '/ops/integrations/advisory-vex-sources'],
['/security/posture', 'Open reachability coverage board', '/security/reachability'],
['/security/advisories-vex', 'Providers', '/security/advisories-vex?tab=providers'],
['/security/advisories-vex', 'VEX Library', '/security/advisories-vex?tab=vex-library'],
['/security/advisories-vex', 'Issuer Trust', '/security/advisories-vex?tab=issuer-trust'],
['/security/disposition', 'Conflicts', '/security/disposition?tab=conflicts'],
['/security/supply-chain-data', 'SBOM Graph', '/security/supply-chain-data/graph'],
['/security/supply-chain-data', 'Reachability', '/security/reachability'],
['/evidence/overview', 'Audit Log', '/evidence/audit-log'],
['/evidence/overview', 'Export Center', '/evidence/exports'],
['/evidence/overview', 'Replay & Verify', '/evidence/verify-replay'],
['/evidence/audit-log', 'View All Events', '/evidence/audit-log/events'],
['/evidence/audit-log', 'Export', '/evidence/exports'],
['/evidence/audit-log', /Policy Audit/i, '/evidence/audit-log/policy'],
['/ops/operations/health-slo', 'View Full Timeline', '/ops/operations/health-slo/incidents'],
['/ops/operations/feeds-airgap', 'Configure Sources', '/ops/integrations/advisory-vex-sources'],
['/ops/operations/feeds-airgap', 'Open Offline Bundles', '/ops/operations/offline-kit/bundles'],
['/ops/operations/feeds-airgap', 'Version Locks', '/ops/operations/feeds-airgap?tenant=demo-prod&regions=us-east&environments=stage&timeWindow=7d&tab=version-locks'],
['/ops/operations/data-integrity', 'Feeds Freshness WARN Impact: BLOCKING NVD feed stale by 3h 12m', '/ops/operations/data-integrity/feeds-freshness'],
['/ops/operations/data-integrity', 'Hotfix 1.2.4', '/releases/approvals?releaseId=rel-hotfix-124'],
['/ops/operations/jobengine', 'Scheduler Runs', '/ops/operations/scheduler/runs'],
['/ops/operations/jobengine', /Execution Quotas/i, '/ops/operations/jobengine/quotas'],
['/ops/operations/offline-kit', 'Bundles', '/ops/operations/offline-kit/bundles'],
['/ops/operations/offline-kit', 'JWKS', '/ops/operations/offline-kit/jwks'],
];
const buttonChecks = [
['/releases/versions', 'Create Release Version', '/releases/versions/new'],
['/releases/versions', 'Create Hotfix Run', '/releases/versions/new'],
['/releases/versions', 'Search'],
['/releases/versions', 'Clear'],
['/releases/investigation/timeline', 'Export'],
['/releases/investigation/change-trace', 'Export'],
['/security/sbom-lake', 'Refresh'],
['/security/sbom-lake', 'Clear'],
['/security/reachability', 'Witnesses'],
['/security/reachability', /PoE|Proof of Exposure/i],
['/evidence/capsules', 'Search'],
['/evidence/threads', 'Search'],
['/evidence/proofs', 'Search'],
['/ops/operations/system-health', 'Services'],
['/ops/operations/system-health', 'Incidents'],
['/ops/operations/system-health', 'Quick Diagnostics'],
['/ops/operations/scheduler', 'Manage Schedules', '/ops/operations/scheduler/schedules'],
['/ops/operations/scheduler', 'Worker Fleet', '/ops/operations/scheduler/workers'],
['/ops/operations/doctor', 'Quick Check'],
['/ops/operations/signals', 'Refresh'],
['/ops/operations/packs', 'Refresh'],
['/ops/operations/status', 'Refresh'],
];
function buildUrl(route) {
const separator = route.includes('?') ? '&' : '?';
return `${baseUrl}${route}${separator}${scopeQuery}`;
}
async function settle(page) {
await page.waitForLoadState('domcontentloaded', { timeout: 15_000 }).catch(() => {});
await page.waitForTimeout(1_500);
}
async function captureSnapshot(page, label) {
const alerts = await page
.locator('[role="alert"], .mat-mdc-snack-bar-container, .toast, .notification, .error-banner, .warning-banner')
.evaluateAll((nodes) =>
nodes
.map((node) => (node.textContent || '').trim().replace(/\s+/g, ' '))
.filter(Boolean)
.slice(0, 5),
)
.catch(() => []);
return {
label,
url: page.url(),
title: await page.title().catch(() => ''),
heading: await page.locator('h1, h2, [data-testid="page-title"], .page-title').first().innerText().catch(() => ''),
alerts,
};
}
async function navigate(page, route) {
await page.goto(buildUrl(route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
await settle(page);
}
async function findLink(page, name, timeoutMs = 10_000) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const link = page.getByRole('link', { name }).first();
if (await link.count()) {
return link;
}
await page.waitForTimeout(250);
}
return null;
}
function matchesLocatorName(name, text) {
if (!text) {
return false;
}
const normalizedText = text.trim();
if (!normalizedText) {
return false;
}
if (name instanceof RegExp) {
return name.test(normalizedText);
}
return normalizedText.includes(name);
}
async function findScopedLink(page, scopeSelector, name) {
const scope = page.locator(scopeSelector).first();
if (!(await scope.count())) {
return null;
}
const descendantLink = scope.getByRole('link', { name }).first();
if (await descendantLink.count()) {
return descendantLink;
}
const [tagName, href, text] = await Promise.all([
scope.evaluate((element) => element.tagName.toLowerCase()).catch(() => ''),
scope.getAttribute('href').catch(() => null),
scope.textContent().catch(() => ''),
]);
if (tagName === 'a' && href && matchesLocatorName(name, text ?? '')) {
return scope;
}
return null;
}
async function findLinkForRoute(page, route, name, timeoutMs = 10_000) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
let locator = null;
if (route === '/releases/environments' && name === 'Open Environment') {
locator = page.locator('.actions').getByRole('link', { name }).first();
} else if (route === '/ops/operations/jobengine' && name instanceof RegExp && String(name) === String(/Execution Quotas/i)) {
locator = await findScopedLink(page, '[data-testid="jobengine-quotas-card"]', name);
} else if (route === '/ops/operations/offline-kit' && name === 'Bundles') {
locator = page.locator('.tab-nav').getByRole('link', { name }).first();
} else {
locator = page.getByRole('link', { name }).first();
}
if (await locator.count()) {
return locator;
}
await page.waitForTimeout(250);
}
return null;
}
async function findButton(page, name, timeoutMs = 10_000) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
for (const role of ['button', 'tab']) {
const button = page.getByRole(role, { name }).first();
if (await button.count()) {
return button;
}
}
await page.waitForTimeout(250);
}
return null;
}
function normalizeUrl(url) {
return decodeURIComponent(url);
}
function shouldIgnoreConsoleError(message) {
return message === 'Failed to load resource: the server responded with a status of 401 ()';
}
function shouldIgnoreRequestFailure(request) {
return request.failure === 'net::ERR_ABORTED';
}
function shouldIgnoreResponseError(response) {
return response.status === 401 && /\/doctor\/api\/v1\/doctor\/run\/[^/]+\/stream$/i.test(response.url);
}
async function runLinkCheck(page, route, name, expectedPath) {
const action = `${route} -> link:${name}`;
try {
await navigate(page, route);
const link = await findLinkForRoute(page, route, name);
if (
!link &&
route === '/ops/operations/jobengine' &&
name instanceof RegExp &&
String(name) === String(/Execution Quotas/i)
) {
const quotasCardText = await page.locator('[data-testid="jobengine-quotas-card"]').textContent().catch(() => '');
const snapshot = await captureSnapshot(page, action);
return {
action,
ok: /access required to manage quotas/i.test(quotasCardText ?? ''),
expectedPath,
finalUrl: normalizeUrl(snapshot.url),
snapshot,
reason: 'restricted-card',
};
}
if (!link) {
return { action, ok: false, reason: 'missing-link', snapshot: await captureSnapshot(page, action) };
}
await link.click({ timeout: 10_000 });
await settle(page);
const snapshot = await captureSnapshot(page, action);
const finalUrl = normalizeUrl(snapshot.url);
return {
action,
ok: finalUrl.includes(expectedPath),
expectedPath,
finalUrl,
snapshot,
};
} catch (error) {
return {
action,
ok: false,
reason: 'exception',
error: error instanceof Error ? error.message : String(error),
snapshot: await captureSnapshot(page, action),
};
}
}
async function runButtonCheck(page, route, name, expectedPath = null) {
const action = `${route} -> button:${name}`;
try {
await navigate(page, route);
const button = await findButton(page, name);
if (!button) {
return { action, ok: false, reason: 'missing-button', snapshot: await captureSnapshot(page, action) };
}
const disabled = await button.isDisabled().catch(() => false);
if (disabled) {
const snapshot = await captureSnapshot(page, action);
return {
action,
ok: true,
reason: 'disabled-by-design',
expectedPath,
finalUrl: normalizeUrl(snapshot.url),
snapshot,
};
}
await button.click({ timeout: 10_000 });
await settle(page);
const snapshot = await captureSnapshot(page, action);
const finalUrl = normalizeUrl(snapshot.url);
const hasRuntimeAlert = snapshot.alerts.some((text) => /(error|failed|unable|timed out|unavailable)/i.test(text));
return {
action,
ok: expectedPath ? finalUrl.includes(expectedPath) : snapshot.heading.trim().length > 0 && !hasRuntimeAlert,
expectedPath,
finalUrl,
snapshot,
};
} catch (error) {
return {
action,
ok: false,
reason: 'exception',
error: error instanceof Error ? error.message : String(error),
snapshot: await captureSnapshot(page, action),
};
}
}
async function main() {
await mkdir(outputDir, { recursive: true });
const browser = await chromium.launch({
headless: process.env.PLAYWRIGHT_HEADLESS !== 'false',
});
const auth = await authenticateFrontdoor({ statePath: authStatePath, reportPath: authReportPath });
const context = await createAuthenticatedContext(browser, auth, { statePath: auth.statePath });
const page = await context.newPage();
const results = [];
const runtime = {
consoleErrors: [],
pageErrors: [],
requestFailures: [],
responseErrors: [],
};
page.on('console', (message) => {
if (message.type() === 'error' && !shouldIgnoreConsoleError(message.text())) {
runtime.consoleErrors.push(message.text());
}
});
page.on('pageerror', (error) => runtime.pageErrors.push(error.message));
page.on('requestfailed', (request) => {
const failure = request.failure()?.errorText ?? 'unknown';
if (!shouldIgnoreRequestFailure({ url: request.url(), failure })) {
runtime.requestFailures.push({ url: request.url(), failure });
}
});
page.on('response', (response) => {
if (response.status() >= 400) {
const candidate = { url: response.url(), status: response.status() };
if (!shouldIgnoreResponseError(candidate)) {
runtime.responseErrors.push(candidate);
}
}
});
try {
for (const [route, name, expectedPath] of linkChecks) {
process.stdout.write(`[live-uncovered-surface-action-sweep] START ${route} -> link:${name}\n`);
const result = await runLinkCheck(page, route, name, expectedPath);
process.stdout.write(
`[live-uncovered-surface-action-sweep] DONE ${route} -> link:${name} ok=${result.ok}\n`,
);
results.push(result);
}
for (const [route, name, expectedPath] of buttonChecks) {
process.stdout.write(`[live-uncovered-surface-action-sweep] START ${route} -> button:${name}\n`);
const result = await runButtonCheck(page, route, name, expectedPath);
process.stdout.write(
`[live-uncovered-surface-action-sweep] DONE ${route} -> button:${name} ok=${result.ok}\n`,
);
results.push(result);
}
} finally {
await page.close().catch(() => {});
await context.close().catch(() => {});
await browser.close().catch(() => {});
}
const failedActions = results.filter((result) => !result.ok);
const report = {
generatedAtUtc: new Date().toISOString(),
baseUrl,
actionCount: results.length,
passedActionCount: results.length - failedActions.length,
failedActionCount: failedActions.length,
failedActions,
results,
runtime,
runtimeIssueCount:
runtime.consoleErrors.length + runtime.pageErrors.length + runtime.requestFailures.length + runtime.responseErrors.length,
};
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
if (failedActions.length > 0 || report.runtimeIssueCount > 0) {
process.exitCode = 1;
}
}
await main();

View File

@@ -0,0 +1,548 @@
#!/usr/bin/env node
import { mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { chromium } from 'playwright';
import { authenticateFrontdoor, createAuthenticatedContext } from './live-frontdoor-auth.mjs';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const webRoot = path.resolve(__dirname, '..');
const outputDir = path.join(webRoot, 'output', 'playwright');
const outputPath = path.join(outputDir, 'live-user-reported-admin-trust-check.json');
const authStatePath = path.join(outputDir, 'live-user-reported-admin-trust-check.state.json');
const authReportPath = path.join(outputDir, 'live-user-reported-admin-trust-check.auth.json');
const baseUrl = process.env.STELLAOPS_FRONTDOOR_BASE_URL?.trim() || 'https://stella-ops.local';
const scopeQuery = 'tenant=demo-prod&regions=us-east&environments=stage&timeWindow=7d';
function buildUrl(route) {
const separator = route.includes('?') ? '&' : '?';
return `${baseUrl}${route}${separator}${scopeQuery}`;
}
async function settle(page, ms = 1500) {
await page.waitForLoadState('domcontentloaded', { timeout: 15_000 }).catch(() => {});
await page.waitForTimeout(ms);
}
async function navigate(page, route) {
await page.goto(buildUrl(route), { waitUntil: 'domcontentloaded', timeout: 30_000 });
await settle(page);
}
async function snapshot(page, label) {
const heading = await page.locator('h1, h2, [data-testid="page-title"], .page-title').first().innerText().catch(() => '');
const alerts = await page
.locator('[role="alert"], .alert, .error-banner, .success-banner, .loading-text, .trust-admin__error, .trust-admin__loading')
.evaluateAll((nodes) => nodes.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim()).filter(Boolean).slice(0, 8))
.catch(() => []);
return {
label,
url: page.url(),
title: await page.title().catch(() => ''),
heading,
alerts,
};
}
function boxesOverlap(left, right) {
if (!left || !right) {
return null;
}
return !(
left.x + left.width <= right.x ||
right.x + right.width <= left.x ||
left.y + left.height <= right.y ||
right.y + right.height <= left.y
);
}
async function collectTrustTabState(page, tab) {
const tableRowCount = await page.locator('tbody tr').count().catch(() => 0);
const eventCardCount = await page.locator('.event-card').count().catch(() => 0);
const emptyTexts = await page
.locator('.key-dashboard__empty, .issuer-trust__empty, .certificate-inventory__empty, .trust-audit-log__empty, .empty-state')
.evaluateAll((nodes) => nodes.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim()).filter(Boolean).slice(0, 6))
.catch(() => []);
const loadingTexts = await page
.locator('.key-dashboard__loading, .trust-admin__loading, .issuer-trust__loading, .certificate-inventory__loading, .trust-audit-log__loading, .loading-text')
.evaluateAll((nodes) => nodes.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim()).filter(Boolean).slice(0, 6))
.catch(() => []);
const primaryButtons = await page
.locator('button')
.evaluateAll((nodes) =>
nodes
.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim())
.filter(Boolean)
.slice(0, 8),
)
.catch(() => []);
return {
tab,
tableRowCount,
eventCardCount,
emptyTexts,
loadingTexts,
primaryButtons,
};
}
async function createRoleCheck(page) {
const roleName = `qa-role-${Date.now()}`;
await page.getByRole('button', { name: '+ Create Role' }).click();
await settle(page, 750);
await page.locator('input[placeholder="security-analyst"]').fill(roleName);
await page.locator('input[placeholder="Security analyst with triage access"]').fill('QA-created role');
await page.locator('textarea[placeholder*="findings:read"]').fill('findings:read, vex:read');
await page.getByRole('button', { name: 'Create Role', exact: true }).click();
await settle(page, 1250);
const successText = await page.locator('.success-banner').first().textContent().then((text) => text?.trim() || '').catch(() => '');
const tableContainsRole = await page.locator('tbody tr td:first-child').evaluateAll(
(cells, expected) => cells.some((cell) => (cell.textContent || '').replace(/\s+/g, ' ').trim() === expected),
roleName,
).catch(() => false);
return {
roleName,
successText,
tableContainsRole,
};
}
async function createTenantCheck(page) {
const tenantId = `qa-tenant-${Date.now()}`;
const displayName = `QA Tenant ${Date.now()}`;
await page.getByRole('button', { name: '+ Add Tenant' }).click();
await settle(page, 750);
await page.locator('input[placeholder="customer-stage"]').fill(tenantId);
await page.locator('input[placeholder="Customer Stage"]').fill(displayName);
await page.locator('select').last().selectOption('shared').catch(() => {});
await page.getByRole('button', { name: 'Create Tenant', exact: true }).click();
await settle(page, 1250);
const successText = await page.locator('.success-banner').first().textContent().then((text) => text?.trim() || '').catch(() => '');
const tableContainsTenant = await page.locator('tbody tr td:first-child').evaluateAll(
(cells, expected) => cells.some((cell) => (cell.textContent || '').replace(/\s+/g, ' ').trim() === expected),
displayName,
).catch(() => false);
return {
tenantId,
displayName,
successText,
tableContainsTenant,
};
}
async function collectReportsTabState(page, tab) {
await page.getByRole('tab', { name: tab }).click();
await settle(page, 1000);
return {
tab,
url: page.url(),
headings: await page.locator('h1, h2, h3').evaluateAll((nodes) =>
nodes.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim()).filter(Boolean).slice(0, 16)
).catch(() => []),
primaryButtons: await page.locator('main button').evaluateAll((nodes) =>
nodes.map((node) => (node.textContent || '').replace(/\s+/g, ' ').trim()).filter(Boolean).slice(0, 12)
).catch(() => []),
};
}
async function runSearchQueryCheck(page, query) {
const searchInput = page.locator('input[aria-label="Global search"]').first();
const responses = [];
const responseListener = async (response) => {
if (!response.url().includes('/api/v1/search/query')) {
return;
}
try {
responses.push({
status: response.status(),
url: response.url(),
body: await response.json(),
});
} catch {
responses.push({
status: response.status(),
url: response.url(),
body: null,
});
}
};
page.on('response', responseListener);
try {
await searchInput.click();
await searchInput.fill(query);
await settle(page, 2500);
const cards = await page.locator('.entity-card').evaluateAll((nodes) =>
nodes.slice(0, 5).map((node) => {
const title = node.querySelector('.entity-card__title')?.textContent?.trim() || '';
const domain = node.querySelector('.entity-card__badge')?.textContent?.trim() || '';
const snippet = node.querySelector('.entity-card__snippet')?.textContent?.replace(/\s+/g, ' ').trim() || '';
const actions = Array.from(node.querySelectorAll('.entity-card__action'))
.map((button) => button.textContent?.replace(/\s+/g, ' ').trim() || '')
.filter(Boolean);
return { title, domain, snippet, actions };
}),
).catch(() => []);
const firstPrimaryAction = page.locator('.entity-card').first().locator('.entity-card__action--primary').first();
const primaryActionLabel = await firstPrimaryAction.textContent().then((text) => text?.replace(/\s+/g, ' ').trim() || '').catch(() => '');
await firstPrimaryAction.click({ timeout: 10000 }).catch(() => {});
await settle(page, 1500);
const latestResponse = responses.at(-1)?.body;
return {
query,
cards,
primaryActionLabel,
latestDiagnostics: latestResponse?.diagnostics ?? null,
latestCardActions: (latestResponse?.cards ?? []).slice(0, 3).map((card) => ({
title: card.title ?? '',
domain: card.domain ?? '',
actions: (card.actions ?? []).map((action) => ({
label: action.label ?? '',
actionType: action.actionType ?? '',
route: action.route ?? '',
})),
})),
finalUrl: page.url(),
snapshot: await snapshot(page, `search:${query}`),
};
} finally {
page.off('response', responseListener);
}
}
async function main() {
await mkdir(outputDir, { recursive: true });
const browser = await chromium.launch({ headless: process.env.PLAYWRIGHT_HEADLESS !== 'false' });
const auth = await authenticateFrontdoor({ statePath: authStatePath, reportPath: authReportPath });
const context = await createAuthenticatedContext(browser, auth, { statePath: auth.statePath });
const page = await context.newPage();
const results = [];
try {
console.log('[live-user-reported-admin-trust-check] sidebar');
await navigate(page, '/setup/identity-access');
const setupGroupContainsDiagnostics = await page
.locator('[role="group"][aria-label="Platform & Setup"]')
.getByText('Diagnostics', { exact: true })
.count()
.then((count) => count > 0)
.catch(() => false);
results.push({
action: 'sidebar:diagnostics-under-setup',
setupGroupContainsDiagnostics,
snapshot: await snapshot(page, 'sidebar:diagnostics-under-setup'),
});
console.log('[live-user-reported-admin-trust-check] invalid-email');
await page.getByRole('button', { name: 'Users' }).click();
await settle(page, 500);
await page.getByRole('button', { name: '+ Add User' }).click();
await settle(page, 500);
const emailInput = page.locator('input[type="email"]').first();
await page.locator('input[placeholder="e.g. jane.doe"]').fill(`qa-user-${Date.now()}`);
await emailInput.fill('not-an-email');
await page.locator('input[placeholder="Jane Doe"]').fill('QA Invalid Email');
const emailValidity = await emailInput.evaluate((input) => ({
valid: input.checkValidity(),
validationMessage: input.validationMessage,
}));
await page.getByRole('button', { name: 'Create User' }).click();
await settle(page, 1000);
results.push({
action: 'identity-access:create-user-invalid-email',
emailValidity,
snapshot: await snapshot(page, 'identity-access:create-user-invalid-email'),
});
console.log('[live-user-reported-admin-trust-check] roles');
await page.getByRole('button', { name: 'Roles' }).click();
await settle(page, 1000);
const roleNames = await page.locator('tbody tr td:first-child').evaluateAll((cells) =>
cells.map((cell) => (cell.textContent || '').replace(/\s+/g, ' ').trim()).filter((value) => value.length > 0),
).catch(() => []);
const roleCreate = await createRoleCheck(page);
results.push({
action: 'identity-access:roles-tab',
roleNames,
roleCreate,
snapshot: await snapshot(page, 'identity-access:roles-tab'),
});
console.log('[live-user-reported-admin-trust-check] tenants');
await page.getByRole('button', { name: 'Tenants' }).click();
await settle(page, 1000);
const tenantCreate = await createTenantCheck(page);
results.push({
action: 'identity-access:tenants-tab',
tenantCreate,
snapshot: await snapshot(page, 'identity-access:tenants-tab'),
});
console.log('[live-user-reported-admin-trust-check] trust-tabs');
await navigate(page, '/setup/trust-signing');
for (const tab of ['Signing Keys', 'Trusted Issuers', 'Certificates', 'Audit Log']) {
await page.getByRole('tab', { name: tab }).click();
await settle(page, 1500);
results.push({
action: `trust-signing:${tab}`,
detail: await collectTrustTabState(page, tab),
snapshot: await snapshot(page, `trust-signing:${tab}`),
});
}
console.log('[live-user-reported-admin-trust-check] reports-tabs');
await navigate(page, '/security/reports');
results.push({
action: 'security-reports:tabs-embedded',
detail: [
await collectReportsTabState(page, 'Risk Report'),
await collectReportsTabState(page, 'VEX Ledger'),
await collectReportsTabState(page, 'Evidence Export'),
],
snapshot: await snapshot(page, 'security-reports:tabs-embedded'),
});
console.log('[live-user-reported-admin-trust-check] triage');
await navigate(page, '/security/triage');
const triageRawSvgTextVisible = await page.locator('main').innerText().then((text) => /<svg|stroke-width|viewBox=/.test(text)).catch(() => false);
results.push({
action: 'security-triage:raw-svg-visible',
triageRawSvgTextVisible,
snapshot: await snapshot(page, 'security-triage:raw-svg-visible'),
});
console.log('[live-user-reported-admin-trust-check] decision-capsules');
await navigate(page, '/evidence/capsules');
const capsulesSearchInput = page.locator('.filter-bar__search-input').first();
const capsulesSearchButton = page.locator('.filter-bar__search-btn').first();
const capsulesSearchInputBox = await capsulesSearchInput.boundingBox().catch(() => null);
const capsulesSearchButtonBox = await capsulesSearchButton.boundingBox().catch(() => null);
results.push({
action: 'decision-capsules:search-layout',
overlaps: boxesOverlap(capsulesSearchInputBox, capsulesSearchButtonBox),
inputBox: capsulesSearchInputBox,
buttonBox: capsulesSearchButtonBox,
snapshot: await snapshot(page, 'decision-capsules:search-layout'),
});
console.log('[live-user-reported-admin-trust-check] global-search');
await navigate(page, '/mission-control/board');
results.push({
action: 'global-search:cve',
detail: await runSearchQueryCheck(page, 'cve'),
});
console.log('[live-user-reported-admin-trust-check] docs');
await navigate(page, '/docs/modules/platform/architecture-overview.md');
results.push({
action: 'docs:architecture-overview',
snapshot: await snapshot(page, 'docs:architecture-overview'),
bodyPreview: await page.locator('.docs-viewer__content').innerText().then((text) => text.slice(0, 240)).catch(() => ''),
});
console.log('[live-user-reported-admin-trust-check] branding');
await navigate(page, '/setup/tenant-branding');
const titleInput = page.locator('#title');
const applyButton = page.getByRole('button', { name: 'Apply Changes' });
const tokenKeyInput = page.locator('input[placeholder="--theme-custom-color"]').first();
const tokenValueInput = page.locator('input[placeholder="var(--color-text-heading)"]').first();
const updatedTitle = `Stella Ops QA ${Date.now()}`;
await titleInput.fill(updatedTitle).catch(() => {});
await settle(page, 300);
const applyDisabled = await applyButton.isDisabled().catch(() => null);
await tokenKeyInput.fill(`--theme-qa-${Date.now()}`).catch(() => {});
await tokenValueInput.fill('#123456').catch(() => {});
await settle(page, 200);
const addTokenButton = page.getByRole('button', { name: 'Add Token' });
const addTokenDisabled = await addTokenButton.isDisabled().catch(() => null);
await addTokenButton.click().catch(() => {});
await settle(page, 500);
const tokenFormCleared =
(await tokenKeyInput.inputValue().catch(() => '')) === '' &&
(await tokenValueInput.inputValue().catch(() => '')) === '';
let applyResult = {
successText: '',
errorText: '',
url: page.url(),
freshAuthPrompt: '',
};
let persistedBranding = {
titleValue: '',
matchesUpdatedTitle: false,
};
if (applyDisabled === false) {
console.log('[live-user-reported-admin-trust-check] branding-apply');
page.once('dialog', async (dialog) => {
applyResult.freshAuthPrompt = dialog.message();
await dialog.accept().catch(() => {});
});
await applyButton.click().catch(() => {});
await settle(page, 2500);
applyResult = {
successText: await page.locator('.success, .success-banner, .alert-success').first().textContent().then((text) => text?.trim() || '').catch(() => ''),
errorText: await page.locator('.error, .error-banner, .alert-error').first().textContent().then((text) => text?.trim() || '').catch(() => ''),
url: page.url(),
freshAuthPrompt: applyResult.freshAuthPrompt,
};
await page.waitForTimeout(2500);
await navigate(page, '/setup/tenant-branding');
persistedBranding = {
titleValue: await page.locator('#title').inputValue().catch(() => ''),
matchesUpdatedTitle: (await page.locator('#title').inputValue().catch(() => '')) === updatedTitle,
};
}
results.push({
action: 'tenant-branding:action-state',
applyDisabled,
addTokenDisabled,
tokenFormCleared,
applyResult,
persistedBranding,
snapshot: await snapshot(page, 'tenant-branding:action-state'),
});
} finally {
console.log('[live-user-reported-admin-trust-check] cleanup');
await page.close().catch(() => {});
await context.close().catch(() => {});
await browser.close().catch(() => {});
}
const report = {
generatedAtUtc: new Date().toISOString(),
baseUrl,
results,
};
const failures = [];
const byAction = new Map(results.map((entry) => [entry.action, entry]));
if (!byAction.get('sidebar:diagnostics-under-setup')?.setupGroupContainsDiagnostics) {
failures.push('Diagnostics is not grouped under Platform & Setup in the sidebar.');
}
if (byAction.get('identity-access:create-user-invalid-email')?.emailValidity?.valid !== false) {
failures.push('Identity & Access user creation did not reject an invalid email address.');
}
if ((byAction.get('identity-access:roles-tab')?.roleNames?.length ?? 0) === 0) {
failures.push('Identity & Access roles table still shows empty role names.');
}
if (!byAction.get('identity-access:roles-tab')?.roleCreate?.tableContainsRole) {
failures.push('Identity & Access role creation did not persist a new role in the table.');
}
if (!byAction.get('identity-access:tenants-tab')?.tenantCreate?.tableContainsTenant) {
failures.push('Identity & Access tenant creation did not persist a new tenant in the table.');
}
for (const tab of ['Signing Keys', 'Trusted Issuers', 'Certificates', 'Audit Log']) {
const detail = byAction.get(`trust-signing:${tab}`)?.detail;
const resolved =
(detail?.tableRowCount ?? 0) > 0 ||
(detail?.eventCardCount ?? 0) > 0 ||
(detail?.emptyTexts?.length ?? 0) > 0;
const stillLoading = (detail?.loadingTexts?.length ?? 0) > 0;
if (!resolved || stillLoading) {
failures.push(`Trust & Signing tab "${tab}" did not resolve cleanly.`);
}
}
for (const tabState of byAction.get('security-reports:tabs-embedded')?.detail ?? []) {
if (!tabState.url.includes('/security/reports')) {
failures.push(`Security Reports tab "${tabState.tab}" still navigates away instead of embedding its workspace.`);
}
}
if (byAction.get('security-triage:raw-svg-visible')?.triageRawSvgTextVisible) {
failures.push('Triage still renders SVG markup as raw code.');
}
if (byAction.get('decision-capsules:search-layout')?.overlaps !== false) {
failures.push('Decision Capsules search input still overlaps the search button.');
}
const searchDetail = byAction.get('global-search:cve')?.detail;
const topCardDomain = (searchDetail?.cards?.[0]?.domain ?? '').toLowerCase();
const topCardTitle = searchDetail?.cards?.[0]?.title ?? '';
const topCardActionLabels = (searchDetail?.latestCardActions?.[0]?.actions ?? []).map((action) => action.label);
const topCardRoute = (searchDetail?.latestCardActions?.[0]?.actions ?? [])[0]?.route ?? '';
const topCardHasDeadNavigation = (searchDetail?.latestCardActions?.[0]?.actions ?? [])
.some((action) => action.actionType === 'navigate' && !action.route);
const searchLandedOnDashboardFallback =
searchDetail?.snapshot?.title === 'Dashboard - StellaOps' ||
searchDetail?.snapshot?.heading === 'Dashboard';
const searchLandedOnBlankDocs =
searchDetail?.finalUrl?.includes('/docs/') &&
!(searchDetail?.snapshot?.heading ?? '').trim();
if (
topCardDomain.includes('api') ||
topCardDomain.includes('knowledge') ||
topCardActionLabels.includes('Copy Curl') ||
topCardHasDeadNavigation ||
searchLandedOnDashboardFallback ||
searchLandedOnBlankDocs
) {
failures.push(`Global search still misranks generic CVE queries or routes them into dead docs/dashboard fallback (top card: ${topCardTitle || 'unknown'}).`);
}
const docsRoute = byAction.get('docs:architecture-overview');
if (!(docsRoute?.snapshot?.heading ?? '').trim() || !(docsRoute?.bodyPreview ?? '').trim()) {
failures.push('Direct /docs navigation still renders an empty or blank documentation page.');
}
if (byAction.get('tenant-branding:action-state')?.applyDisabled !== false) {
failures.push('Tenant & Branding apply action did not become available after editing.');
}
if (byAction.get('tenant-branding:action-state')?.tokenFormCleared !== true) {
failures.push('Tenant & Branding add-token action did not complete cleanly.');
}
const brandingApplyResult = byAction.get('tenant-branding:action-state')?.applyResult;
if (
byAction.get('tenant-branding:action-state')?.applyDisabled === false &&
!brandingApplyResult?.successText &&
!brandingApplyResult?.errorText
) {
failures.push('Tenant & Branding apply action did not produce a visible success or error outcome.');
}
if (
byAction.get('tenant-branding:action-state')?.applyDisabled === false &&
byAction.get('tenant-branding:action-state')?.persistedBranding?.matchesUpdatedTitle !== true
) {
failures.push('Tenant & Branding apply action did not persist the updated title after reload.');
}
report.failures = failures;
report.failedCheckCount = failures.length;
report.ok = failures.length === 0;
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
if (failures.length > 0) {
process.exitCode = 1;
}
}
await main();