Harden scratch-stack live QA sweeps

2026-03-11 12:07:00 +02:00
parent 568a1df468
commit 4dc5db4efb
4 changed files with 238 additions and 19 deletions
--- a/src/Web/StellaOps.Web/scripts/live-frontdoor-unified-search-route-matrix.mjs
+++ b/src/Web/StellaOps.Web/scripts/live-frontdoor-unified-search-route-matrix.mjs
@@ -48,6 +48,8 @@ function createRuntime() {
  return {
    consoleErrors: [],
    pageErrors: [],
+    requestFailures: [],
+    responseErrors: [],
  };
 }

@@ -69,6 +71,43 @@ function attachRuntimeListeners(page, runtime) {
      message: error.message,
    });
  });
+
+  page.on('requestfailed', (request) => {
+    const url = request.url();
+    if (/\.(?:css|js|map|png|jpg|jpeg|svg|woff2?)(?:$|\?)/i.test(url)) {
+      return;
+    }
+
+    const errorText = request.failure()?.errorText ?? 'unknown';
+    if (errorText === 'net::ERR_ABORTED') {
+      return;
+    }
+
+    runtime.requestFailures.push({
+      timestamp: Date.now(),
+      page: page.url(),
+      method: request.method(),
+      url,
+      error: errorText,
+    });
+  });
+
+  page.on('response', (response) => {
+    const url = response.url();
+    if (!url.includes('/api/v1/search')) {
+      return;
+    }
+
+    if (response.status() >= 400) {
+      runtime.responseErrors.push({
+        timestamp: Date.now(),
+        page: page.url(),
+        method: response.request().method(),
+        status: response.status(),
+        url,
+      });
+    }
+  });
 }

 async function readVisibleTexts(locator) {
@@ -88,7 +127,45 @@ async function openSearch(page, route) {
  const input = page.locator('app-global-search input[type="text"]').first();
  await input.click({ timeout: 10_000 });
  await page.waitForSelector('.search__results', { state: 'visible', timeout: 10_000 });
-  await page.waitForTimeout(4_000);
+  await waitForStarterPanel(page);
+}
+
+async function waitForStarterPanel(page, timeoutMs = 20_000) {
+  const startedAt = Date.now();
+
+  while (Date.now() - startedAt < timeoutMs) {
+    const state = await page.evaluate(() => {
+      const starterChips = document.querySelectorAll('.search__suggestions .search__chip').length;
+      const degradedBanners = document.querySelectorAll('.search__degraded-banner').length;
+      const emptyTexts = Array.from(document.querySelectorAll('.search__empty, .search__empty-state-copy'))
+        .map((node) => (node.textContent || '').trim().replace(/\s+/g, ' '))
+        .filter((text) => text.length > 0);
+
+      return {
+        starterChips,
+        degradedBanners,
+        emptyTexts,
+      };
+    }).catch(() => ({
+      starterChips: 0,
+      degradedBanners: 0,
+      emptyTexts: [],
+    }));
+
+    if (state.starterChips > 0 || state.degradedBanners > 0) {
+      await page.waitForTimeout(750);
+      return;
+    }
+
+    const hasDefaultPromptOnly = state.emptyTexts.length > 0
+      && state.emptyTexts.every((text) => /ask about what is on this page/i.test(text));
+    if (!hasDefaultPromptOnly && state.emptyTexts.length > 0) {
+      await page.waitForTimeout(750);
+      return;
+    }
+
+    await page.waitForTimeout(500);
+  }
 }

 async function captureSnapshot(page, routeConfig, runtime, routeStartedAt) {
@@ -107,6 +184,8 @@ async function captureSnapshot(page, routeConfig, runtime, routeStartedAt) {
    cardTitles: await readVisibleTexts(page.locator('.search__cards .entity-card__title')),
    consoleErrors: runtime.consoleErrors.filter((entry) => entry.timestamp >= routeStartedAt),
    pageErrors: runtime.pageErrors.filter((entry) => entry.timestamp >= routeStartedAt),
+    requestFailures: runtime.requestFailures.filter((entry) => entry.timestamp >= routeStartedAt),
+    responseErrors: runtime.responseErrors.filter((entry) => entry.timestamp >= routeStartedAt),
  };
 }

@@ -154,6 +233,16 @@ function buildIssues(routeResult) {
  }
  issues.push(...routeResult.snapshot.consoleErrors.map((entry) => `console:${entry.text}`));
  issues.push(...routeResult.snapshot.pageErrors.map((entry) => `pageerror:${entry.message}`));
+  issues.push(
+    ...routeResult.snapshot.requestFailures.map(
+      (entry) => `requestfailed:${entry.method} ${entry.url} ${entry.error}`,
+    ),
+  );
+  issues.push(
+    ...routeResult.snapshot.responseErrors.map(
+      (entry) => `response:${entry.status} ${entry.method} ${entry.url}`,
+    ),
+  );
  for (const starter of routeResult.executedStarters) {
    if (!starter.ok) {
      issues.push(`Starter index ${starter.starterIndex} "${starter.starterText}" did not resolve to grounded results on ${routeResult.route}`);
--- a/src/Web/StellaOps.Web/scripts/live-integrations-action-sweep.mjs
+++ b/src/Web/StellaOps.Web/scripts/live-integrations-action-sweep.mjs
@@ -16,7 +16,7 @@ const STEP_TIMEOUT_MS = 30_000;

 async function settle(page) {
  await page.waitForLoadState('domcontentloaded', { timeout: 15_000 }).catch(() => {});
-  await page.waitForTimeout(1_500);
+  await page.waitForTimeout(2_500);
 }

 async function headingText(page) {
@@ -65,6 +65,51 @@ async function navigate(page, route) {
  return url;
 }

+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+async function resolveInteractiveLocator(page, name, role = 'link', timeoutMs = 8_000) {
+  const matcher = name instanceof RegExp ? name : new RegExp(escapeRegExp(name), 'i');
+  const startedAt = Date.now();
+
+  while (Date.now() - startedAt < timeoutMs) {
+    const candidates = [
+      page.getByRole(role, { name: matcher }),
+      page.locator(role === 'tab' ? '[role="tab"]' : 'a, [role="link"]').filter({ hasText: matcher }),
+    ];
+
+    for (const candidate of candidates) {
+      if ((await candidate.count().catch(() => 0)) > 0) {
+        const locator = candidate.first();
+        if (await locator.isVisible().catch(() => true)) {
+          return locator;
+        }
+      }
+    }
+
+    await page.waitForTimeout(250);
+  }
+
+  return null;
+}
+
+async function resolveButtonLocator(page, name, timeoutMs = 8_000) {
+  const matcher = name instanceof RegExp ? name : new RegExp(escapeRegExp(name), 'i');
+  const startedAt = Date.now();
+
+  while (Date.now() - startedAt < timeoutMs) {
+    const locator = page.getByRole('button', { name: matcher }).first();
+    if ((await locator.count().catch(() => 0)) > 0 && (await locator.isVisible().catch(() => true))) {
+      return locator;
+    }
+
+    await page.waitForTimeout(250);
+  }
+
+  return null;
+}
+
 async function runAction(page, route, label, runner) {
  const startedAtUtc = new Date().toISOString();
  const startedAt = Date.now();
@@ -117,14 +162,14 @@ async function runAction(page, route, label, runner) {
 async function clickLinkVerify(page, route, name, expectedPath) {
  await navigate(page, route);
  const candidates = [
-    page.getByRole('link', { name }),
-    page.getByRole('tab', { name }),
+    await resolveInteractiveLocator(page, name, 'link'),
+    await resolveInteractiveLocator(page, name, 'tab'),
  ];

  let locator = null;
  for (const candidate of candidates) {
-    if ((await candidate.count()) > 0) {
-      locator = candidate.first();
+    if (candidate) {
+      locator = candidate;
      break;
    }
  }
@@ -150,8 +195,8 @@ async function clickLinkVerify(page, route, name, expectedPath) {

 async function clickButtonVerify(page, route, name, expectedPath) {
  await navigate(page, route);
-  const locator = page.getByRole('button', { name }).first();
-  if ((await locator.count()) === 0) {
+  const locator = await resolveButtonLocator(page, name);
+  if (!locator) {
    return {
      ok: false,
      reason: 'missing-button',
@@ -170,6 +215,62 @@ async function clickButtonVerify(page, route, name, expectedPath) {
  };
 }

+async function clickEmptyStateOrFirstDetailVerify(page, route, buttonName, expectedPath) {
+  await navigate(page, route);
+
+  const emptyStateButton = await resolveButtonLocator(page, buttonName, 4_000);
+  if (emptyStateButton) {
+    await emptyStateButton.click({ timeout: 10_000 });
+    await page.waitForURL((url) => url.pathname.includes(expectedPath), { timeout: 15_000 });
+    await settle(page);
+
+    return {
+      ok: page.url().includes(expectedPath),
+      mode: 'empty-state-button',
+      expectedPath,
+      snapshot: await captureSnapshot(page, `after-empty-state-button:${buttonName}`),
+    };
+  }
+
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < 8_000) {
+    const detailLink = page.locator('tbody tr td a').first();
+    if ((await detailLink.count().catch(() => 0)) > 0 && (await detailLink.isVisible().catch(() => true))) {
+      const originPath = new URL(page.url()).pathname;
+      await detailLink.click({ timeout: 10_000 });
+      await page.waitForURL(
+        (url) => url.pathname !== originPath && url.pathname.startsWith('/ops/integrations/'),
+        { timeout: 15_000 },
+      );
+      await settle(page);
+
+      return {
+        ok: true,
+        mode: 'detail-link',
+        expectedPath: '/ops/integrations/:integrationId',
+        snapshot: await captureSnapshot(page, `after-detail-link:${route}`),
+      };
+    }
+
+    const loadError = page.locator('.error-state').first();
+    if ((await loadError.count().catch(() => 0)) > 0 && (await loadError.isVisible().catch(() => true))) {
+      return {
+        ok: false,
+        reason: 'load-error',
+        snapshot: await captureSnapshot(page, `load-error:${route}`),
+      };
+    }
+
+    await page.waitForTimeout(250);
+  }
+
+  return {
+    ok: false,
+    reason: 'missing-empty-state-and-detail',
+    snapshot: await captureSnapshot(page, `missing-empty-state-and-detail:${route}`),
+  };
+}
+
 async function main() {
  await mkdir(outputDir, { recursive: true });

@@ -255,8 +356,10 @@ async function main() {
          clickLinkVerify(page, '/ops/integrations', 'CI/CD', '/ops/integrations/ci')),
        await runAction(page, '/ops/integrations', 'link:Runtimes / Hosts', () =>
          clickLinkVerify(page, '/ops/integrations', 'Runtimes / Hosts', '/ops/integrations/runtime-hosts')),
-        await runAction(page, '/ops/integrations', 'link:Advisory & VEX', () =>
-          clickLinkVerify(page, '/ops/integrations', 'Advisory & VEX', '/ops/integrations/advisory-vex-sources')),
+        await runAction(page, '/ops/integrations', 'link:Advisory Sources', () =>
+          clickLinkVerify(page, '/ops/integrations', 'Advisory Sources', '/ops/integrations/advisory-vex-sources')),
+        await runAction(page, '/ops/integrations', 'link:VEX Sources', () =>
+          clickLinkVerify(page, '/ops/integrations', 'VEX Sources', '/ops/integrations/advisory-vex-sources')),
        await runAction(page, '/ops/integrations', 'link:Secrets', () =>
          clickLinkVerify(page, '/ops/integrations', 'Secrets', '/ops/integrations/secrets')),
        await runAction(page, '/ops/integrations', 'button:+ Add Integration', () =>
@@ -272,8 +375,13 @@ async function main() {
      actions: [
        await runAction(page, '/ops/integrations/registries', 'button:+ Add Registry', () =>
          clickButtonVerify(page, '/ops/integrations/registries', '+ Add Registry', '/ops/integrations/onboarding/registry')),
-        await runAction(page, '/ops/integrations/registries', 'button:Add your first registry', () =>
-          clickButtonVerify(page, '/ops/integrations/registries', 'Add your first registry', '/ops/integrations/onboarding/registry')),
+        await runAction(page, '/ops/integrations/registries', 'empty-or-detail:Registries', () =>
+          clickEmptyStateOrFirstDetailVerify(
+            page,
+            '/ops/integrations/registries',
+            'Add your first registry',
+            '/ops/integrations/onboarding/registry',
+          )),
      ],
    });
    await persistSummary(summary);
@@ -283,8 +391,13 @@ async function main() {
      actions: [
        await runAction(page, '/ops/integrations/runtime-hosts', 'button:+ Add RuntimeHost', () =>
          clickButtonVerify(page, '/ops/integrations/runtime-hosts', '+ Add RuntimeHost', '/ops/integrations/onboarding/host')),
-        await runAction(page, '/ops/integrations/runtime-hosts', 'button:Add your first runtimehost', () =>
-          clickButtonVerify(page, '/ops/integrations/runtime-hosts', 'Add your first runtimehost', '/ops/integrations/onboarding/host')),
+        await runAction(page, '/ops/integrations/runtime-hosts', 'empty-or-detail:Runtimes / Hosts', () =>
+          clickEmptyStateOrFirstDetailVerify(
+            page,
+            '/ops/integrations/runtime-hosts',
+            'Add your first runtimehost',
+            '/ops/integrations/onboarding/host',
+          )),
      ],
    });
    await persistSummary(summary);
@@ -294,8 +407,13 @@ async function main() {
      actions: [
        await runAction(page, '/ops/integrations/secrets', 'button:+ Add Integration', () =>
          clickButtonVerify(page, '/ops/integrations/secrets', '+ Add Integration', '/ops/integrations/onboarding')),
-        await runAction(page, '/ops/integrations/secrets', 'button:Open add integration hub', () =>
-          clickButtonVerify(page, '/ops/integrations/secrets', 'Open add integration hub', '/ops/integrations/onboarding')),
+        await runAction(page, '/ops/integrations/secrets', 'empty-or-detail:Secrets', () =>
+          clickEmptyStateOrFirstDetailVerify(
+            page,
+            '/ops/integrations/secrets',
+            'Open add integration hub',
+            '/ops/integrations/onboarding',
+          )),
      ],
    });
    await persistSummary(summary);
@@ -305,8 +423,13 @@ async function main() {
      actions: [
        await runAction(page, '/ops/integrations/advisory-vex-sources', 'button:+ Add Integration', () =>
          clickButtonVerify(page, '/ops/integrations/advisory-vex-sources', '+ Add Integration', '/ops/integrations/onboarding')),
-        await runAction(page, '/ops/integrations/advisory-vex-sources', 'button:Open add integration hub', () =>
-          clickButtonVerify(page, '/ops/integrations/advisory-vex-sources', 'Open add integration hub', '/ops/integrations/onboarding')),
+        await runAction(page, '/ops/integrations/advisory-vex-sources', 'empty-or-detail:Advisory & VEX Sources', () =>
+          clickEmptyStateOrFirstDetailVerify(
+            page,
+            '/ops/integrations/advisory-vex-sources',
+            'Open add integration hub',
+            '/ops/integrations/onboarding',
+          )),
      ],
    });
    await persistSummary(summary);