Close scratch iteration 006 grouped readiness repairs

src/Web/StellaOps.Web/scripts/live-frontdoor-authenticated-readiness.mjs

@@ -0,0 +1,318 @@
+#!/usr/bin/env node
+
+import { mkdirSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { chromium } from 'playwright';
+
+import { authenticateFrontdoor, createAuthenticatedContext } from './live-frontdoor-auth.mjs';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const webRoot = path.resolve(__dirname, '..');
+const outputDirectory = path.join(webRoot, 'output', 'playwright');
+const statePath = path.join(outputDirectory, 'live-frontdoor-authenticated-readiness.state.json');
+const reportPath = path.join(outputDirectory, 'live-frontdoor-authenticated-readiness.auth.json');
+const resultPath = path.join(outputDirectory, 'live-frontdoor-authenticated-readiness.json');
+
+const baseUrl = process.env.STELLAOPS_FRONTDOOR_BASE_URL?.trim() || 'https://stella-ops.local';
+const maxAttempts = Number.parseInt(process.env.STELLAOPS_READY_ATTEMPTS ?? '18', 10);
+const retryDelayMs = Number.parseInt(process.env.STELLAOPS_READY_RETRY_DELAY_MS ?? '5000', 10);
+const scopeQuery = 'tenant=demo-prod&regions=us-east&environments=stage&timeWindow=7d';
+
+function isStaticAsset(url) {
+  return /\.(?:css|js|map|png|jpg|jpeg|svg|woff2?|ico)(?:$|\?)/i.test(url);
+}
+
+function isAbortedNavigationFailure(errorText) {
+  return typeof errorText === 'string' && /aborted|net::err_abort/i.test(errorText);
+}
+
+function createRuntime() {
+  return {
+    consoleErrors: [],
+    pageErrors: [],
+    requestFailures: [],
+    responseErrors: [],
+  };
+}
+
+async function waitForPageReady(page, route, titleFragment, markers, options = {}) {
+  const {
+    requireAllMarkers = false,
+    forbiddenMarkers = [],
+  } = options;
+  await page.goto(`${baseUrl}${route}`, {
+    waitUntil: 'domcontentloaded',
+    timeout: 30_000,
+  });
+
+  await page.waitForLoadState('domcontentloaded', { timeout: 15_000 }).catch(() => {});
+  await page.waitForTimeout(1_000);
+
+  const loadingText = page.locator('text=/Loading /i').first();
+  await loadingText.waitFor({ state: 'hidden', timeout: 10_000 }).catch(() => {});
+
+  const expectedPath = new URL(`${baseUrl}${route}`).pathname;
+  let lastSnapshot = null;
+  for (let attempt = 0; attempt < 30; attempt += 1) {
+    lastSnapshot = await page.evaluate(({ expectedTitle, expectedMarkers, expectedForbiddenMarkers }) => {
+      const title = document.title.trim();
+      const bodyText = document.body?.innerText?.replace(/\s+/g, ' ') ?? '';
+      return {
+        path: window.location.pathname,
+        title,
+        matchedTitle: !expectedTitle || title.includes(expectedTitle),
+        matchedMarkers: expectedMarkers.filter((marker) => bodyText.includes(marker)),
+        forbiddenMarkers: expectedForbiddenMarkers.filter((marker) => bodyText.includes(marker)),
+      };
+    }, {
+      expectedTitle: titleFragment,
+      expectedMarkers: markers,
+      expectedForbiddenMarkers: forbiddenMarkers,
+    });
+
+    if (
+      lastSnapshot.path === expectedPath &&
+      lastSnapshot.matchedTitle &&
+      (
+        requireAllMarkers
+          ? lastSnapshot.matchedMarkers.length === markers.length
+          : lastSnapshot.matchedMarkers.length > 0
+      ) &&
+      lastSnapshot.forbiddenMarkers.length === 0
+    ) {
+      await page.waitForTimeout(500);
+      return;
+    }
+
+    await page.waitForTimeout(1_000);
+  }
+
+  throw new Error(
+    `Route ${route} did not become ready. ` +
+    `path=${lastSnapshot?.path ?? '<unknown>'} ` +
+    `title=${lastSnapshot?.title ?? '<empty>'} ` +
+    `matchedMarkers=${(lastSnapshot?.matchedMarkers ?? []).join(',') || '<none>'} ` +
+    `forbiddenMarkers=${(lastSnapshot?.forbiddenMarkers ?? []).join(',') || '<none>'}`,
+  );
+}
+
+async function waitForPromotionPreview(page) {
+  const previewSignals = [
+    page.locator('.preview-summary').first(),
+    page.getByText('All gates passed').first(),
+    page.getByText('One or more gates are not passing').first(),
+    page.getByText('Required approvers:').first(),
+  ];
+
+  await Promise.any(
+    previewSignals.map((locator) => locator.waitFor({ state: 'visible', timeout: 30_000 })),
+  );
+}
+
+async function warmTopology(page) {
+  await waitForPageReady(page, `/setup/topology/environments?${scopeQuery}`, 'Environments', [
+    'Environment Inventory',
+    'Region-first',
+    'SEARCH',
+  ]);
+}
+
+async function warmPromotion(page) {
+  await waitForPageReady(page, `/releases/promotions/create?${scopeQuery}`, 'Create Promotion', [
+    'Create Promotion',
+    'Load Target Environments',
+  ]);
+
+  await page.getByLabel('Release/Bundle identity').fill('rel-001');
+  await page.getByRole('button', { name: 'Load Target Environments' }).click({ timeout: 10_000 });
+
+  await page.waitForFunction(() => {
+    const select = document.querySelector('#target-env');
+    if (!(select instanceof HTMLSelectElement)) {
+      return false;
+    }
+
+    return [...select.options].some((option) => option.value === 'env-staging');
+  }, { timeout: 30_000 });
+
+  await page.locator('#target-env').selectOption('env-staging');
+
+  const previewButton = page.getByRole('button', { name: 'Refresh Gate Preview' });
+  await previewButton.waitFor({ state: 'visible', timeout: 30_000 });
+  await previewButton.click({ timeout: 10_000 });
+  await waitForPromotionPreview(page);
+}
+
+async function warmNotifications(page) {
+  await waitForPageReady(
+    page,
+    `/setup/notifications/config/overrides?${scopeQuery}`,
+    'Notifications',
+    [
+      'Notification Administration',
+      'Operator Overrides',
+    ],
+    {
+      requireAllMarkers: true,
+      forbiddenMarkers: [
+        'Failed to load overrides',
+      ],
+    },
+  );
+}
+
+async function main() {
+  mkdirSync(outputDirectory, { recursive: true });
+
+  const authReport = await authenticateFrontdoor({
+    statePath,
+    reportPath,
+    headless: true,
+  });
+
+  const browser = await chromium.launch({
+    headless: true,
+    args: ['--disable-dev-shm-usage'],
+  });
+
+  const context = await createAuthenticatedContext(browser, authReport, { statePath });
+  const page = await context.newPage();
+
+  let runtime = createRuntime();
+
+  page.on('console', (message) => {
+    if (message.type() === 'error') {
+      runtime.consoleErrors.push({
+        page: page.url(),
+        text: message.text(),
+      });
+    }
+  });
+
+  page.on('pageerror', (error) => {
+    if (page.url() === 'about:blank' && String(error).includes('sessionStorage')) {
+      return;
+    }
+
+    runtime.pageErrors.push({
+      page: page.url(),
+      text: error instanceof Error ? error.message : String(error),
+    });
+  });
+
+  page.on('requestfailed', (request) => {
+    const errorText = request.failure()?.errorText ?? 'unknown';
+    if (isStaticAsset(request.url()) || isAbortedNavigationFailure(errorText)) {
+      return;
+    }
+
+    runtime.requestFailures.push({
+      page: page.url(),
+      method: request.method(),
+      url: request.url(),
+      error: errorText,
+    });
+  });
+
+  page.on('response', (response) => {
+    if (isStaticAsset(response.url()) || response.status() < 400) {
+      return;
+    }
+
+    runtime.responseErrors.push({
+      page: page.url(),
+      method: response.request().method(),
+      status: response.status(),
+      url: response.url(),
+    });
+  });
+
+  const summary = {
+    checkedAtUtc: new Date().toISOString(),
+    baseUrl,
+    maxAttempts,
+    retryDelayMs,
+    attempts: [],
+    finalStatus: 'failed',
+  };
+
+  try {
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+      runtime = createRuntime();
+
+      const attemptResult = {
+        attempt,
+        startedAtUtc: new Date().toISOString(),
+        topologyReady: false,
+        promotionReady: false,
+        notificationsReady: false,
+        runtime,
+        ok: false,
+      };
+
+      try {
+        await warmTopology(page);
+        attemptResult.topologyReady = true;
+
+        await warmPromotion(page);
+        attemptResult.promotionReady = true;
+
+        await warmNotifications(page);
+        attemptResult.notificationsReady = true;
+      } catch (error) {
+        attemptResult.error = error instanceof Error ? error.message : String(error);
+      }
+
+      attemptResult.ok =
+        attemptResult.topologyReady &&
+        attemptResult.promotionReady &&
+        attemptResult.notificationsReady &&
+        runtime.consoleErrors.length === 0 &&
+        runtime.pageErrors.length === 0 &&
+        runtime.requestFailures.length === 0 &&
+        runtime.responseErrors.length === 0;
+
+      summary.attempts.push(attemptResult);
+      summary.finalStatus = attemptResult.ok ? 'ready' : 'retrying';
+      writeFileSync(resultPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
+
+      if (attemptResult.ok) {
+        summary.finalStatus = 'ready';
+        break;
+      }
+
+      if (attempt < maxAttempts) {
+        await page.goto('about:blank').catch(() => {});
+        await page.waitForTimeout(retryDelayMs);
+      }
+    }
+  } finally {
+    writeFileSync(resultPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
+    await context.close();
+    await browser.close();
+  }
+
+  const latestAttempt = summary.attempts.at(-1);
+  if (!latestAttempt?.ok) {
+    summary.finalStatus = 'failed';
+    writeFileSync(resultPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
+    const failureMessage = latestAttempt?.error
+      ?? [
+        ...latestAttempt.runtime.responseErrors.map((entry) => `${entry.method} ${entry.url} -> ${entry.status}`),
+        ...latestAttempt.runtime.requestFailures.map((entry) => `${entry.method} ${entry.url} failed: ${entry.error}`),
+        ...latestAttempt.runtime.consoleErrors.map((entry) => entry.text),
+        ...latestAttempt.runtime.pageErrors.map((entry) => entry.text),
+      ].join('; ');
+    throw new Error(failureMessage || 'Authenticated frontdoor readiness did not converge.');
+  }
+
+  process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
+}
+
+main().catch((error) => {
+  process.stderr.write(`[live-frontdoor-authenticated-readiness] ${error instanceof Error ? error.message : String(error)}\n`);
+  process.exit(1);
+});

src/Web/StellaOps.Web/scripts/live-notifications-watchlist-recheck.mjs

@@ -21,6 +21,30 @@ async function settle(page) {
   await page.waitForTimeout(1_500);
 }
 
+async function waitForNotificationsPanel(page, timeoutMs = 12_000) {
+  await page.waitForFunction(() => {
+    if (document.querySelector('.notify-panel')) {
+      return true;
+    }
+
+    return Array.from(document.querySelectorAll('h1, h2, [data-testid="page-title"], .page-title'))
+      .map((node) => (node.textContent || '').trim().toLowerCase())
+      .some((text) => text.includes('notify control plane') || text === 'channels' || text === 'rules');
+  }, { timeout: timeoutMs }).catch(() => {});
+  await page.waitForTimeout(500);
+}
+
+async function waitForAdminNotifications(page, timeoutMs = 12_000) {
+  await page.waitForFunction(() => {
+    const bodyText = document.body?.innerText?.replace(/\s+/g, ' ') ?? '';
+    const hasHeading = Array.from(document.querySelectorAll('h1, h2, [data-testid="page-title"], .page-title'))
+      .map((node) => (node.textContent || '').trim())
+      .some((text) => text === 'Notification Administration');
+    return hasHeading && bodyText.includes('Notification Administration');
+  }, { timeout: timeoutMs }).catch(() => {});
+  await page.waitForTimeout(500);
+}
+
 async function headingText(page) {
   const headings = page.locator('h1, h2, [data-testid="page-title"], .page-title');
   const count = await headings.count();
@@ -59,6 +83,12 @@ async function navigate(page, route) {
   const url = `https://stella-ops.local${route}${separator}${scopeQuery}`;
   await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 });
   await settle(page);
+  if (route === '/ops/operations/notifications') {
+    await waitForNotificationsPanel(page);
+  }
+  if (route.startsWith('/setup/notifications')) {
+    await waitForAdminNotifications(page);
+  }
   return url;
 }
 
@@ -171,11 +201,16 @@ async function main() {
       return;
     }
 
+    const errorText = request.failure()?.errorText ?? 'unknown';
+    if (errorText === 'net::ERR_ABORTED') {
+      return;
+    }
+
     runtime.requestFailures.push({
       page: page.url(),
       method: request.method(),
       url,
-      error: request.failure()?.errorText ?? 'unknown',
+      error: errorText,
     });
   });
   page.on('response', (response) => {

src/Web/StellaOps.Web/scripts/live-release-promotion-submit-check.mjs

@@ -53,7 +53,27 @@ function collectScopeIssues(label, targetUrl) {
 }
 
 async function clickNext(page) {
-  await page.getByRole('button', { name: 'Next ->' }).click();
+  await clickStableButton(page, 'Next ->');
+}
+
+async function waitForSection(page, headingText) {
+  await page.getByRole('heading', { name: headingText, exact: true }).waitFor({
+    state: 'visible',
+    timeout: 20_000,
+  });
+}
+
+async function waitForPromotionPreview(page) {
+  const previewSignals = [
+    page.locator('.preview-summary').first(),
+    page.getByText('All gates passed').first(),
+    page.getByText('One or more gates are not passing').first(),
+    page.getByText('Required approvers:').first(),
+  ];
+
+  await Promise.any(
+    previewSignals.map((locator) => locator.waitFor({ state: 'visible', timeout: 30_000 })),
+  );
 }
 
 async function clickStableButton(page, name) {
@@ -176,13 +196,25 @@ async function main() {
       timeout: 30_000,
     });
 
+    await waitForSection(page, 'Select Bundle Version Identity');
     await page.getByLabel('Release/Bundle identity').fill('rel-001');
     await page.getByRole('button', { name: 'Load Target Environments' }).click();
+    await waitForSection(page, 'Select Region and Environment Path');
+    await page.locator('#target-env').waitFor({ state: 'visible', timeout: 20_000 });
+    await page.waitForFunction(() => {
+      const select = document.querySelector('#target-env');
+      return select instanceof HTMLSelectElement
+        && [...select.options].some((option) => option.value === 'env-staging');
+    }, { timeout: 30_000 });
     await page.locator('#target-env').selectOption('env-staging');
-    await page.getByRole('button', { name: 'Refresh Gate Preview' }).click();
+    await waitForSection(page, 'Gate Preview');
+    await clickStableButton(page, 'Refresh Gate Preview');
+    await waitForPromotionPreview(page);
     await clickNext(page);
+    await waitForSection(page, 'Approval Context');
     await page.getByLabel('Justification').fill('Release approval path validated end to end.');
     await clickNext(page);
+    await waitForSection(page, 'Launch Promotion');
 
     const submitResponsePromise = page.waitForResponse(
       (response) =>