Standardize live search Playwright setup lane

This commit is contained in:
master
2026-03-08 11:17:05 +02:00
parent 6870649abf
commit e01a499df9
10 changed files with 692 additions and 51 deletions

View File

@@ -0,0 +1,40 @@
import { expect, test } from '@playwright/test';
import {
ensureLiveServiceHealthy,
rebuildLiveIndexes,
resolveLiveSearchRuntimeConfig,
runGroundedSmokeQuery,
runSourcesPrepare,
writeLiveSearchSetupArtifact,
} from './support/live-search-support';
test.describe('Live search setup', () => {
test('prepares the ingested corpus and proves grounded retrieval before browser execution', async ({}, testInfo) => {
testInfo.setTimeout(300_000);
const config = resolveLiveSearchRuntimeConfig();
await ensureLiveServiceHealthy(config, 180_000);
const sourcesPrepare = runSourcesPrepare(config);
const { knowledge, unified } = await rebuildLiveIndexes(config);
const smokeQuery = await runGroundedSmokeQuery(config);
const artifactPath = writeLiveSearchSetupArtifact(config, {
preparedAtUtc: new Date().toISOString(),
baseUrl: config.baseUrl,
tenant: config.tenant,
sourcesPrepare,
knowledgeRebuild: knowledge,
unifiedRebuild: unified,
smokeQuery,
});
expect((smokeQuery['contextAnswer'] as Record<string, unknown> | undefined)?.['status']).toBe('grounded');
expect(Array.isArray(smokeQuery['cards']) ? smokeQuery['cards'].length : 0).toBeGreaterThan(0);
await testInfo.attach('live-search-setup.json', {
path: artifactPath,
contentType: 'application/json',
});
});
});

View File

@@ -0,0 +1,267 @@
import { execFileSync } from 'node:child_process';
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
import path from 'node:path';
export type LiveSearchRuntimeConfig = {
baseUrl: string;
tenant: string;
scopes: string;
actor: string;
advisoryConnectionString: string;
repoRoot: string;
webRoot: string;
};
export type LiveSearchSetupReport = {
preparedAtUtc: string;
baseUrl: string;
tenant: string;
sourcesPrepare: Record<string, unknown>;
knowledgeRebuild: Record<string, unknown>;
unifiedRebuild: Record<string, unknown>;
smokeQuery: Record<string, unknown>;
};
const DEFAULT_LIVE_SEARCH_BASE_URL = 'http://127.0.0.1:10451';
const DEFAULT_LIVE_TENANT = 'test-tenant';
const DEFAULT_LIVE_SCOPES = 'advisory-ai:view advisory-ai:operate advisory-ai:admin';
const DEFAULT_LIVE_ACTOR = 'playwright-live-setup';
const DEFAULT_ADVISORY_CONNSTRING =
'Host=localhost;Port=55432;Database=advisoryai_knowledge_test;Username=stellaops_knowledge;Password=stellaops_knowledge';
export function resolveLiveSearchRuntimeConfig(): LiveSearchRuntimeConfig {
const repoRoot = findRepoRoot(process.cwd());
const webRoot = path.join(repoRoot, 'src', 'Web', 'StellaOps.Web');
return {
baseUrl: process.env['LIVE_ADVISORYAI_SEARCH_BASE_URL']?.trim() || DEFAULT_LIVE_SEARCH_BASE_URL,
tenant: process.env['LIVE_ADVISORYAI_TENANT']?.trim() || DEFAULT_LIVE_TENANT,
scopes: process.env['LIVE_ADVISORYAI_SCOPES']?.trim() || DEFAULT_LIVE_SCOPES,
actor: process.env['LIVE_ADVISORYAI_ACTOR']?.trim() || DEFAULT_LIVE_ACTOR,
advisoryConnectionString:
process.env['AdvisoryAI__KnowledgeSearch__ConnectionString']?.trim() || DEFAULT_ADVISORY_CONNSTRING,
repoRoot,
webRoot,
};
}
export async function ensureLiveServiceHealthy(
config: LiveSearchRuntimeConfig,
timeoutMs = 120_000,
): Promise<void> {
const start = Date.now();
let lastError = 'service did not respond';
while (Date.now() - start < timeoutMs) {
try {
const response = await fetch(`${config.baseUrl}/health`);
if (response.ok) {
return;
}
lastError = `status ${response.status}`;
} catch (error) {
lastError = error instanceof Error ? error.message : String(error);
}
await delay(1_000);
}
throw new Error(`Live AdvisoryAI health check failed at ${config.baseUrl}/health: ${lastError}`);
}
export function ensureLocalCliPublished(config: LiveSearchRuntimeConfig): string {
const publishDirectory = path.join(config.repoRoot, '.artifacts', 'stella-cli');
const cliBinaryName = process.platform === 'win32' ? 'StellaOps.Cli.exe' : 'StellaOps.Cli';
const cliBinaryPath = path.join(publishDirectory, cliBinaryName);
if (existsSync(cliBinaryPath)) {
return cliBinaryPath;
}
execFileSync(
'dotnet',
[
'publish',
path.join(config.repoRoot, 'src', 'Cli', 'StellaOps.Cli', 'StellaOps.Cli.csproj'),
'-c',
'Release',
'-o',
publishDirectory,
],
{
cwd: config.repoRoot,
env: process.env,
stdio: 'pipe',
},
);
if (!existsSync(cliBinaryPath)) {
throw new Error(`CLI publish completed without producing ${cliBinaryPath}.`);
}
return cliBinaryPath;
}
export function runSourcesPrepare(
config: LiveSearchRuntimeConfig,
cliBinaryPath = ensureLocalCliPublished(config),
): Record<string, unknown> {
const stdout = execFileSync(
cliBinaryPath,
['advisoryai', 'sources', 'prepare', '--json'],
{
cwd: config.repoRoot,
env: {
...process.env,
STELLAOPS_BACKEND_URL: config.baseUrl,
},
stdio: 'pipe',
encoding: 'utf8',
},
);
return parseJsonObject(stdout, 'sources prepare');
}
export async function rebuildLiveIndexes(
config: LiveSearchRuntimeConfig,
): Promise<{ knowledge: Record<string, unknown>; unified: Record<string, unknown> }> {
const adminHeaders = buildLiveHeaders(config, 'advisory-ai:admin');
const knowledge = await postJson(`${config.baseUrl}/v1/advisory-ai/index/rebuild`, adminHeaders);
const unified = await postJson(`${config.baseUrl}/v1/search/index/rebuild`, adminHeaders);
return { knowledge, unified };
}
export async function runGroundedSmokeQuery(
config: LiveSearchRuntimeConfig,
): Promise<Record<string, unknown>> {
return postJson(
`${config.baseUrl}/v1/search/query`,
buildLiveHeaders(config),
{
q: 'database connectivity',
k: 5,
includeSynthesis: false,
ambient: {
currentRoute: '/ops/operations/doctor',
},
},
);
}
export function writeLiveSearchSetupArtifact(
config: LiveSearchRuntimeConfig,
report: LiveSearchSetupReport,
): string {
const outputDirectory = path.join(config.webRoot, 'output', 'playwright');
const outputPath = path.join(outputDirectory, 'live-search-setup.json');
mkdirSync(outputDirectory, { recursive: true });
writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
return outputPath;
}
export function buildLiveHeaders(
config: LiveSearchRuntimeConfig,
scopes = config.scopes,
): Record<string, string> {
return {
'content-type': 'application/json',
'x-stellaops-scopes': scopes,
'x-stellaops-tenant': config.tenant,
'x-stellaops-actor': config.actor,
};
}
async function postJson(
url: string,
headers: Record<string, string>,
body?: Record<string, unknown>,
): Promise<Record<string, unknown>> {
const response = await fetch(url, {
method: 'POST',
headers,
body: body ? JSON.stringify(body) : undefined,
});
const raw = await response.text();
if (!response.ok) {
throw new Error(`Request to ${url} failed with status ${response.status}: ${raw}`);
}
return parseJsonObject(raw, url);
}
function parseJsonObject(raw: string, source: string): Record<string, unknown> {
const trimmed = raw.trim();
if (!trimmed) {
return {};
}
try {
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
return parsed && typeof parsed === 'object' ? parsed : {};
} catch {
const extracted = tryExtractEmbeddedJson(trimmed);
if (extracted) {
return extracted;
}
throw new Error(`Expected JSON output from ${source}, received: ${trimmed.slice(0, 400)}`);
}
}
function tryExtractEmbeddedJson(raw: string): Record<string, unknown> | null {
const firstBrace = raw.indexOf('{');
const lastBrace = raw.lastIndexOf('}');
if (firstBrace >= 0 && lastBrace > firstBrace) {
const candidate = raw.slice(firstBrace, lastBrace + 1);
try {
const parsed = JSON.parse(candidate) as Record<string, unknown>;
return parsed && typeof parsed === 'object' ? parsed : {};
} catch {
// Fall through to narrower candidates.
}
}
for (let index = raw.indexOf('{'); index >= 0; index = raw.indexOf('{', index + 1)) {
try {
const parsed = JSON.parse(raw.slice(index)) as Record<string, unknown>;
return parsed && typeof parsed === 'object' ? parsed : {};
} catch {
// Keep scanning for the start of the JSON payload.
}
}
return null;
}
function findRepoRoot(startDirectory: string): string {
let current = path.resolve(startDirectory);
while (true) {
if (isRepoRoot(current)) {
return current;
}
const parent = path.dirname(current);
if (parent === current) {
throw new Error(`Unable to locate the Stella Ops repository root from ${startDirectory}.`);
}
current = parent;
}
}
function isRepoRoot(candidate: string): boolean {
return existsSync(path.join(candidate, 'global.json'))
&& existsSync(path.join(candidate, 'src'))
&& existsSync(path.join(candidate, 'docs'))
&& existsSync(path.join(candidate, 'devops'));
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

View File

@@ -1,12 +1,18 @@
import { expect, test, type Page, type Route } from '@playwright/test';
import { policyAuthorSession } from '../../src/app/testing';
import {
buildLiveHeaders,
ensureLiveServiceHealthy,
rebuildLiveIndexes,
resolveLiveSearchRuntimeConfig,
} from './support/live-search-support';
import { waitForEntityCards, waitForResults } from './unified-search-fixtures';
const liveSearchBaseUrl = process.env['LIVE_ADVISORYAI_SEARCH_BASE_URL']?.trim() ?? '';
const liveTenant = process.env['LIVE_ADVISORYAI_TENANT']?.trim() || 'test-tenant';
const liveScopes = process.env['LIVE_ADVISORYAI_SCOPES']?.trim()
|| 'advisory-ai:view advisory-ai:operate advisory-ai:admin';
const liveRuntime = resolveLiveSearchRuntimeConfig();
const liveSearchBaseUrl = liveRuntime.baseUrl;
const liveTenant = liveRuntime.tenant;
const liveScopes = liveRuntime.scopes;
const mockConfig = {
authority: {
@@ -162,8 +168,10 @@ test.describe('Unified Search - Live contextual suggestions', () => {
test.beforeAll(async ({}, testInfo) => {
testInfo.setTimeout(120_000);
await ensureLiveServiceHealthy(liveSearchBaseUrl);
await rebuildLiveIndexes(liveSearchBaseUrl);
await ensureLiveServiceHealthy(liveRuntime);
if (process.env['PLAYWRIGHT_LIVE_SEARCH_SKIP_REBUILD'] !== '1') {
await rebuildLiveIndexes(liveRuntime);
}
liveRouteStates.clear();
for (const routeConfig of liveRouteConfigs) {
@@ -319,6 +327,34 @@ test.describe('Unified Search - Live contextual suggestions', () => {
expect(String(capturedTurnBodies.at(-1)?.['content'] ?? '')).toMatch(/grounded answer|best next step/i);
});
test('persists a grounded live search in recent history and clears it with the icon action', async ({ page }) => {
await routeLiveUnifiedSearch(page);
await openDoctor(page);
const searchInput = page.locator('app-global-search input[type="text"]');
await searchInput.focus();
await waitForResults(page);
await page.locator('.search__suggestions .search__chip', {
hasText: /database connectivity/i,
}).first().click();
await expect(searchInput).toHaveValue('database connectivity');
await waitForResults(page);
await expect(page.locator('[data-answer-status="grounded"]')).toBeVisible();
await searchInput.fill('');
await waitForResults(page);
const recentGroup = page.locator('.search__group').filter({ hasText: 'Recent' });
await expect(recentGroup).toContainText('database connectivity');
const clearButton = page.locator('.search__clear-history');
await expect(clearButton).toBeVisible();
await clearButton.click();
await expect(recentGroup).toHaveCount(0);
});
for (const routeConfig of liveRouteConfigs.filter((route) => route.key !== 'doctor')) {
test(`${routeConfig.label} suppresses surfaced starter chips when the live route corpus is unready`, async ({ page }) => {
const state = requireLiveRouteState(routeConfig.key);
@@ -534,6 +570,8 @@ async function routeLiveUnifiedSearch(
page: Page,
capturedRequests?: Array<Record<string, unknown>>,
): Promise<void> {
const liveHeaders = buildLiveHeaders(liveRuntime);
await page.route('**/api/v1/search/query', async (route) => {
const rawBody = route.request().postData() ?? '{}';
const parsedBody = safeParseRequest(rawBody);
@@ -543,12 +581,7 @@ async function routeLiveUnifiedSearch(
const response = await fetch(`${liveSearchBaseUrl}/v1/search/query`, {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-stellaops-scopes': liveScopes,
'x-stellaops-tenant': liveTenant,
'x-stellaops-actor': 'playwright-live',
},
headers: liveHeaders,
body: rawBody,
});
@@ -571,38 +604,6 @@ async function routeLiveUnifiedSearch(
});
}
async function ensureLiveServiceHealthy(baseUrl: string): Promise<void> {
const response = await fetch(`${baseUrl}/health`);
if (!response.ok) {
throw new Error(`Live AdvisoryAI health check failed with status ${response.status}.`);
}
}
async function rebuildLiveIndexes(baseUrl: string): Promise<void> {
const headers = {
'content-type': 'application/json',
'x-stellaops-scopes': 'advisory-ai:admin',
'x-stellaops-tenant': liveTenant,
'x-stellaops-actor': 'playwright-live',
};
const knowledgeResponse = await fetch(`${baseUrl}/v1/advisory-ai/index/rebuild`, {
method: 'POST',
headers,
});
if (!knowledgeResponse.ok) {
throw new Error(`Knowledge rebuild failed with status ${knowledgeResponse.status}.`);
}
const unifiedResponse = await fetch(`${baseUrl}/v1/search/index/rebuild`, {
method: 'POST',
headers,
});
if (!unifiedResponse.ok) {
throw new Error(`Unified rebuild failed with status ${unifiedResponse.status}.`);
}
}
function safeParseRequest(rawBody: string): Record<string, unknown> {
try {
const parsed = JSON.parse(rawBody) as Record<string, unknown>;
@@ -613,12 +614,7 @@ function safeParseRequest(rawBody: string): Record<string, unknown> {
}
async function fetchLiveSuggestionViability(rawBody: string): Promise<Record<string, unknown>> {
const headers = {
'content-type': 'application/json',
'x-stellaops-scopes': liveScopes,
'x-stellaops-tenant': liveTenant,
'x-stellaops-actor': 'playwright-live',
};
const headers = buildLiveHeaders(liveRuntime);
const directResponse = await fetch(`${liveSearchBaseUrl}/v1/search/suggestions/evaluate`, {
method: 'POST',