Verify live search suggestions against ingested corpus

2026-03-07 18:52:18 +02:00
parent 9d3bed1d0e
commit 820fb4ec25
4 changed files with 324 additions and 16 deletions
--- a/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts
+++ b/src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts
@@ -7,6 +7,7 @@ const liveSearchBaseUrl = process.env['LIVE_ADVISORYAI_SEARCH_BASE_URL']?.trim()
 const liveTenant = process.env['LIVE_ADVISORYAI_TENANT']?.trim() || 'test-tenant';
 const liveScopes = process.env['LIVE_ADVISORYAI_SCOPES']?.trim()
  || 'advisory-ai:view advisory-ai:operate advisory-ai:admin';
+const liveSuggestionSeedQueries = ['database connectivity', 'OIDC readiness'];

 const mockConfig = {
  authority: {
@@ -112,13 +113,14 @@ test.describe('Unified Search - Live contextual suggestions', () => {
  test.beforeAll(async () => {
    await ensureLiveServiceHealthy(liveSearchBaseUrl);
    await rebuildLiveIndexes(liveSearchBaseUrl);
+    await assertLiveSuggestionCoverage(liveSearchBaseUrl, liveSuggestionSeedQueries);
  });

  test.beforeEach(async ({ page }) => {
    await setupDoctorPage(page);
  });

-  test('shows automatic suggestion chips when the doctor page opens', async ({ page }) => {
+  test('shows only viable live suggestion chips when the doctor page opens', async ({ page }) => {
    await routeLiveUnifiedSearch(page);
    await openDoctor(page);

@@ -138,6 +140,34 @@ test.describe('Unified Search - Live contextual suggestions', () => {
    }).first()).toBeVisible();
  });

+  test('every surfaced doctor suggestion executes into a grounded or clarify state', async ({ page }) => {
+    await routeLiveUnifiedSearch(page);
+    await openDoctor(page);
+
+    const searchInput = page.locator('app-global-search input[type="text"]');
+    await searchInput.focus();
+    await waitForResults(page);
+
+    const suggestionTexts = (await page.locator('.search__suggestions .search__chip').allTextContents())
+      .map((text) => text.trim())
+      .filter((text) => text.length > 0);
+
+    expect(suggestionTexts.length).toBeGreaterThan(0);
+
+    for (const suggestionText of suggestionTexts) {
+      await openDoctor(page);
+      await searchInput.focus();
+      await waitForResults(page);
+      await page.locator('.search__suggestions .search__chip', {
+        hasText: new RegExp(`^${escapeRegExp(suggestionText)}$`, 'i'),
+      }).first().click();
+
+      await expect(searchInput).toHaveValue(suggestionText);
+      await waitForResults(page);
+      await assertNonDeadEndSearch(page, suggestionText);
+    }
+  });
+
  test('clicking a suggestion chip executes a live query and shows a grounded answer', async ({ page }) => {
    const capturedRequests: Array<Record<string, unknown>> = [];
    await routeLiveUnifiedSearch(page, capturedRequests);
@@ -196,6 +226,31 @@ test.describe('Unified Search - Live contextual suggestions', () => {
      hasText: /follow up:\s*database connectivity/i,
    }).first()).toBeVisible();
  });
+
+  test('answer-panel Ask AdvisoryAI keeps the live query context', async ({ page }) => {
+    const capturedTurnBodies: Array<Record<string, unknown>> = [];
+    await routeLiveUnifiedSearch(page);
+    await mockChatConversation(page, capturedTurnBodies);
+    await openDoctor(page);
+
+    const searchInput = page.locator('app-global-search input[type="text"]');
+    await searchInput.focus();
+    await waitForResults(page);
+    await page.locator('.search__suggestions .search__chip', {
+      hasText: /database connectivity/i,
+    }).first().click();
+
+    await expect(searchInput).toHaveValue('database connectivity');
+    await waitForResults(page);
+    await expect(page.locator('[data-answer-status="grounded"]')).toBeVisible();
+
+    await page.locator('[data-answer-action="ask-ai"]').click();
+
+    await expect(page.locator('.assistant-drawer')).toBeVisible({ timeout: 10_000 });
+    await expect.poll(() => capturedTurnBodies.length).toBeGreaterThan(0);
+    expect(String(capturedTurnBodies.at(-1)?.['content'] ?? '')).toMatch(/database connectivity/i);
+    expect(String(capturedTurnBodies.at(-1)?.['content'] ?? '')).toMatch(/grounded answer|best next step/i);
+  });
 });

 async function setupDoctorPage(page: Page): Promise<void> {
@@ -311,6 +366,16 @@ async function routeLiveUnifiedSearch(
      body,
    });
  });
+
+  await page.route('**/api/v1/search/suggestions/evaluate', async (route) => {
+    const rawBody = route.request().postData() ?? '{}';
+    const body = await fetchLiveSuggestionViability(rawBody);
+    await route.fulfill({
+      status: 200,
+      contentType: 'application/json',
+      body: JSON.stringify(body),
+    });
+  });
 }

 async function ensureLiveServiceHealthy(baseUrl: string): Promise<void> {
@@ -345,6 +410,26 @@ async function rebuildLiveIndexes(baseUrl: string): Promise<void> {
  }
 }

+async function assertLiveSuggestionCoverage(
+  baseUrl: string,
+  queries: readonly string[],
+): Promise<void> {
+  const payload = await fetchLiveSuggestionViability(JSON.stringify({
+    queries,
+    ambient: {
+      currentRoute: '/ops/operations/doctor',
+    },
+  }));
+  const suggestions = Array.isArray(payload['suggestions'])
+    ? payload['suggestions'] as Array<Record<string, unknown>>
+    : [];
+  const viableSuggestions = suggestions.filter((suggestion) => suggestion['viable'] === true);
+
+  if (viableSuggestions.length === 0) {
+    throw new Error(`Live suggestion preflight returned no viable queries: ${JSON.stringify(payload)}`);
+  }
+}
+
 function safeParseRequest(rawBody: string): Record<string, unknown> {
  try {
    const parsed = JSON.parse(rawBody) as Record<string, unknown>;
@@ -353,3 +438,218 @@ function safeParseRequest(rawBody: string): Record<string, unknown> {
    return {};
  }
 }
+
+async function fetchLiveSuggestionViability(rawBody: string): Promise<Record<string, unknown>> {
+  const headers = {
+    'content-type': 'application/json',
+    'x-stellaops-scopes': liveScopes,
+    'x-stellaops-tenant': liveTenant,
+    'x-stellaops-actor': 'playwright-live',
+  };
+
+  const directResponse = await fetch(`${liveSearchBaseUrl}/v1/search/suggestions/evaluate`, {
+    method: 'POST',
+    headers,
+    body: rawBody,
+  });
+
+  if (directResponse.ok) {
+    return safeParseRequest(await directResponse.text());
+  }
+
+  if (directResponse.status !== 404) {
+    throw new Error(`Live suggestion preflight failed with status ${directResponse.status}.`);
+  }
+
+  const parsedBody = safeParseRequest(rawBody);
+  return buildCompatibilitySuggestionViability(parsedBody, headers);
+}
+
+async function buildCompatibilitySuggestionViability(
+  requestBody: Record<string, unknown>,
+  headers: Record<string, string>,
+): Promise<Record<string, unknown>> {
+  const queries = Array.isArray(requestBody['queries'])
+    ? requestBody['queries'].map((query) => String(query ?? '').trim()).filter((query) => query.length > 0)
+    : [];
+  const filters = requestBody['filters'];
+  const ambient = requestBody['ambient'];
+  const suggestions: Array<Record<string, unknown>> = [];
+  let mergedCoverage: Record<string, unknown> | null = null;
+
+  for (const query of queries) {
+    const response = await fetch(`${liveSearchBaseUrl}/v1/search/query`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({
+        q: query,
+        k: 5,
+        includeSynthesis: false,
+        filters,
+        ambient,
+      }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`Compatibility suggestion query failed for "${query}" with status ${response.status}.`);
+    }
+
+    const payload = safeParseRequest(await response.text());
+    const cards = Array.isArray(payload['cards']) ? payload['cards'] as Array<Record<string, unknown>> : [];
+    const overflow = payload['overflow'] && typeof payload['overflow'] === 'object'
+      ? payload['overflow'] as Record<string, unknown>
+      : null;
+    const overflowCards = Array.isArray(overflow?.['cards']) ? overflow!['cards'] as Array<Record<string, unknown>> : [];
+    const contextAnswer = payload['contextAnswer'] && typeof payload['contextAnswer'] === 'object'
+      ? payload['contextAnswer'] as Record<string, unknown>
+      : null;
+    const coverage = payload['coverage'] && typeof payload['coverage'] === 'object'
+      ? payload['coverage'] as Record<string, unknown>
+      : null;
+    const cardCount = cards.length + overflowCards.length;
+    const status = String(contextAnswer?.['status'] ?? 'insufficient');
+    const leadingDomain =
+      String(cards[0]?.['domain'] ?? overflowCards[0]?.['domain'] ?? coverage?.['currentScopeDomain'] ?? '');
+
+    suggestions.push({
+      query,
+      viable: cardCount > 0 || status === 'clarify',
+      status,
+      code: String(contextAnswer?.['code'] ?? 'no_grounded_evidence'),
+      cardCount,
+      leadingDomain: leadingDomain || undefined,
+      reason: String(contextAnswer?.['reason'] ?? 'No grounded evidence matched the suggestion in the active corpus.'),
+    });
+
+    mergedCoverage = mergeCoverage(mergedCoverage, coverage);
+  }
+
+  return {
+    suggestions,
+    coverage: mergedCoverage,
+  };
+}
+
+function mergeCoverage(
+  current: Record<string, unknown> | null,
+  next: Record<string, unknown> | null,
+): Record<string, unknown> | null {
+  if (!next) {
+    return current;
+  }
+
+  if (!current) {
+    return next;
+  }
+
+  const currentDomains = Array.isArray(current['domains']) ? current['domains'] as Array<Record<string, unknown>> : [];
+  const nextDomains = Array.isArray(next['domains']) ? next['domains'] as Array<Record<string, unknown>> : [];
+  const mergedDomainsByKey = new Map<string, Record<string, unknown>>();
+
+  for (const domain of [...currentDomains, ...nextDomains]) {
+    const key = String(domain['domain'] ?? '');
+    if (!key) {
+      continue;
+    }
+
+    const existing = mergedDomainsByKey.get(key);
+    if (!existing) {
+      mergedDomainsByKey.set(key, domain);
+      continue;
+    }
+
+    mergedDomainsByKey.set(key, {
+      domain: key,
+      candidateCount: Math.max(Number(existing['candidateCount'] ?? 0), Number(domain['candidateCount'] ?? 0)),
+      visibleCardCount: Math.max(Number(existing['visibleCardCount'] ?? 0), Number(domain['visibleCardCount'] ?? 0)),
+      topScore: Math.max(Number(existing['topScore'] ?? 0), Number(domain['topScore'] ?? 0)),
+      isCurrentScope: Boolean(existing['isCurrentScope']) || Boolean(domain['isCurrentScope']),
+      hasVisibleResults: Boolean(existing['hasVisibleResults']) || Boolean(domain['hasVisibleResults']),
+    });
+  }
+
+  return {
+    currentScopeDomain: String(current['currentScopeDomain'] ?? next['currentScopeDomain'] ?? ''),
+    currentScopeWeighted: Boolean(current['currentScopeWeighted']) || Boolean(next['currentScopeWeighted']),
+    domains: Array.from(mergedDomainsByKey.values()),
+  };
+}
+
+async function assertNonDeadEndSearch(page: Page, suggestionText: string): Promise<void> {
+  await expect.poll(async () => {
+    const status = await page.locator('[data-answer-status]').first().getAttribute('data-answer-status');
+    if (status === 'grounded' || status === 'clarify') {
+      return status;
+    }
+
+    return '';
+  }, {
+    message: `Expected "${suggestionText}" to resolve into a grounded or clarify answer.`,
+  }).not.toBe('');
+
+  const answerStatus = await page.locator('[data-answer-status]').first().getAttribute('data-answer-status');
+  if (answerStatus === 'grounded') {
+    await waitForEntityCards(page, 1);
+  }
+}
+
+async function mockChatConversation(
+  page: Page,
+  capturedTurnBodies: Array<Record<string, unknown>>,
+): Promise<void> {
+  await page.route('**/api/v1/advisory-ai/conversations', async (route) => {
+    if (route.request().method() !== 'POST') {
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify([]),
+      });
+    }
+
+    return route.fulfill({
+      status: 200,
+      contentType: 'application/json',
+      body: JSON.stringify({
+        conversationId: 'conv-live-context-1',
+        tenantId: liveTenant,
+        userId: 'playwright-live',
+        context: {},
+        turns: [],
+        createdAt: '2026-03-07T00:00:00.000Z',
+        updatedAt: '2026-03-07T00:00:00.000Z',
+      }),
+    });
+  });
+
+  await page.route('**/api/v1/advisory-ai/conversations/*/turns', async (route) => {
+    if (route.request().method() !== 'POST') {
+      return route.continue();
+    }
+
+    capturedTurnBodies.push((route.request().postDataJSON() as Record<string, unknown> | null) ?? {});
+    const events = [
+      'event: progress',
+      'data: {"stage":"searching"}',
+      '',
+      'event: token',
+      'data: {"content":"I can expand the grounded answer and recommend the next step."}',
+      '',
+      'event: done',
+      'data: {"turnId":"turn-live-context-1","groundingScore":0.93}',
+      '',
+    ].join('\n');
+
+    return route.fulfill({
+      status: 200,
+      headers: {
+        'content-type': 'text/event-stream; charset=utf-8',
+        'cache-control': 'no-cache',
+      },
+      body: events,
+    });
+  });
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}