stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 3 Releases Wiki Activity

Verify supported-route live search matrix

Browse Source
This commit is contained in:
master
2026-03-08 02:23:58 +02:00
parent 145e67a544
commit 93872e73ec
3 changed files with 235 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
src/Web/StellaOps.Web/tests/e2e/unified-search-contextual-suggestions.live.e2e.spec.ts
View File

@@ -1,4 +1,4 @@
import { expect, test, type Page } from '@playwright/test';
import { expect, test, type Page, type Route } from '@playwright/test';
import { policyAuthorSession } from '../../src/app/testing';
import { waitForEntityCards, waitForResults } from './unified-search-fixtures';
@@ -7,7 +7,6 @@ const liveSearchBaseUrl = process.env['LIVE_ADVISORYAI_SEARCH_BASE_URL']?.trim()
const liveTenant = process.env['LIVE_ADVISORYAI_TENANT']?.trim() || 'test-tenant';
const liveScopes = process.env['LIVE_ADVISORYAI_SCOPES']?.trim()
|| 'advisory-ai:view advisory-ai:operate advisory-ai:admin';
const liveSuggestionSeedQueries = ['database connectivity', 'OIDC readiness'];
const mockConfig = {
authority: {
@@ -106,6 +105,56 @@ const mockChecks = {
total: 3,
};
type LiveRouteConfig = {
key: string;
label: string;
path: string;
heading: RegExp;
seedQueries: readonly string[];
};
type LiveRouteState = {
config: LiveRouteConfig;
supported: boolean;
scopeReady: boolean;
viableCount: number;
viabilityStates: string[];
payload: Record<string, unknown>;
};
const liveRouteConfigs: readonly LiveRouteConfig[] = [
{
key: 'doctor',
label: 'Doctor',
path: '/ops/operations/doctor',
heading: /doctor diagnostics/i,
seedQueries: ['database connectivity', 'OIDC readiness'],
},
{
key: 'triage',
label: 'Security triage',
path: '/security/triage',
heading: /security\s*\/\s*triage/i,
seedQueries: ['critical findings', 'reachable vulnerabilities', 'unresolved CVEs'],
},
{
key: 'policy',
label: 'Policy governance',
path: '/ops/policy',
heading: /policy/i,
seedQueries: ['failing policy gates', 'production deny rules', 'policy exceptions'],
},
{
key: 'vex',
label: 'Advisories and VEX',
path: '/security/advisories-vex',
heading: /advisories\s*&\s*vex/i,
seedQueries: ['Why is this marked not affected?', 'Which components are covered by this VEX?', 'What evidence conflicts with this VEX?'],
},
] as const;
const liveRouteStates = new Map<string, LiveRouteState>();
test.describe('Unified Search - Live contextual suggestions', () => {
test.describe.configure({ mode: 'serial' });
test.setTimeout(120_000);
@@ -115,11 +164,20 @@ test.describe('Unified Search - Live contextual suggestions', () => {
testInfo.setTimeout(120_000);
await ensureLiveServiceHealthy(liveSearchBaseUrl);
await rebuildLiveIndexes(liveSearchBaseUrl);
await assertLiveSuggestionCoverage(liveSearchBaseUrl, liveSuggestionSeedQueries);
liveRouteStates.clear();
for (const routeConfig of liveRouteConfigs) {
liveRouteStates.set(routeConfig.key, await evaluateLiveRoute(routeConfig));
}
const doctorState = requireLiveRouteState('doctor');
if (!doctorState.supported) {
throw new Error(`Doctor live suggestion preflight returned no grounded viable suggestions: ${JSON.stringify(doctorState.payload)}`);
}
});
test.beforeEach(async ({ page }) => {
await setupDoctorPage(page);
await setupLiveShell(page);
});
test('shows only viable live suggestion chips when the doctor page opens', async ({ page }) => {
@@ -260,9 +318,88 @@ test.describe('Unified Search - Live contextual suggestions', () => {
expect(String(capturedTurnBodies.at(-1)?.['content'] ?? '')).toMatch(/database connectivity/i);
expect(String(capturedTurnBodies.at(-1)?.['content'] ?? '')).toMatch(/grounded answer|best next step/i);
});
for (const routeConfig of liveRouteConfigs.filter((route) => route.key !== 'doctor')) {
test(`${routeConfig.label} suppresses surfaced starter chips when the live route corpus is unready`, async ({ page }) => {
const state = requireLiveRouteState(routeConfig.key);
test.skip(state.supported || state.scopeReady, `${routeConfig.label} live route is no longer corpus-unready.`);
await routeLiveUnifiedSearch(page);
await openLiveRoute(page, routeConfig);
const searchInput = page.locator('app-global-search input[type="text"]');
await searchInput.focus();
await waitForResults(page);
await expect(page.locator('.search__context-title')).toContainText(routeConfig.heading);
await expect(page.locator('.search__suggestions .search__chip')).toHaveCount(0);
await expect(page.locator('.search__group-label', {
hasText: /start here/i,
})).toHaveCount(0);
});
test(`${routeConfig.label} executes every surfaced suggestion when the live route corpus is ready`, async ({ page }) => {
const state = requireLiveRouteState(routeConfig.key);
test.skip(!state.supported, `${routeConfig.label} live route is not yet suggestion-ready.`);
await routeLiveUnifiedSearch(page);
await openLiveRoute(page, routeConfig);
const searchInput = page.locator('app-global-search input[type="text"]');
await searchInput.focus();
await waitForResults(page);
const suggestionChips = page.locator('.search__suggestions .search__chip');
await expect.poll(async () => await suggestionChips.count(), {
message: `${routeConfig.label} should surface at least one viable starter chip.`,
}).toBeGreaterThan(0);
await expect(suggestionChips.first()).toBeVisible({ timeout: 10_000 });
const suggestionTexts = (await suggestionChips.allTextContents())
.map((text) => text.trim())
.filter((text) => text.length > 0);
expect(suggestionTexts.length).toBeGreaterThan(0);
for (const suggestionText of suggestionTexts) {
await openLiveRoute(page, routeConfig);
await searchInput.focus();
await waitForResults(page);
const suggestionChip = page.locator('.search__suggestions .search__chip', {
hasText: new RegExp(`^${escapeRegExp(suggestionText)}$`, 'i'),
}).first();
if (await suggestionChip.isVisible().catch(() => false)) {
await suggestionChip.click();
} else {
await searchInput.fill(suggestionText);
}
await expect(searchInput).toHaveValue(suggestionText);
await waitForResults(page);
await assertGroundedSearch(page, suggestionText);
}
});
}
});
async function setupDoctorPage(page: Page): Promise<void> {
async function setupLiveShell(page: Page): Promise<void> {
const jsonStubUnlessDocument = (defaultGetBody: unknown = []): ((route: Route) => Promise<void>) => {
return async (route) => {
if (route.request().resourceType() === 'document') {
await route.fallback();
return;
}
await route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify(route.request().method() === 'GET' ? defaultGetBody : {}),
});
};
};
await page.addInitScript((stubSession) => {
(window as unknown as { __stellaopsTestSession?: unknown }).__stellaopsTestSession = stubSession;
}, doctorSession);
@@ -337,15 +474,62 @@ async function setupDoctorPage(page: Page): Promise<void> {
}),
}),
);
await page.route('**/api/**', jsonStubUnlessDocument());
await page.route('**/gateway/**', jsonStubUnlessDocument());
await page.route('**/policy/**', jsonStubUnlessDocument());
await page.route('**/scanner/**', jsonStubUnlessDocument());
await page.route('**/concelier/**', jsonStubUnlessDocument());
await page.route('**/attestor/**', jsonStubUnlessDocument());
}
async function openDoctor(page: Page): Promise<void> {
await page.goto('/ops/operations/doctor', { waitUntil: 'domcontentloaded' });
await expect(page.getByRole('heading', { name: /doctor diagnostics/i })).toBeVisible({
await openLiveRoute(page, liveRouteConfigs[0]);
}
async function openLiveRoute(page: Page, routeConfig: LiveRouteConfig): Promise<void> {
await page.goto(routeConfig.path, { waitUntil: 'domcontentloaded' });
await expect(page.getByRole('heading', { name: routeConfig.heading }).first()).toBeVisible({
timeout: 15_000,
});
}
function requireLiveRouteState(routeKey: string): LiveRouteState {
const state = liveRouteStates.get(routeKey);
if (!state) {
throw new Error(`Missing live route preflight for ${routeKey}.`);
}
return state;
}
async function evaluateLiveRoute(routeConfig: LiveRouteConfig): Promise<LiveRouteState> {
const payload = await fetchLiveSuggestionViability(JSON.stringify({
queries: routeConfig.seedQueries,
ambient: {
currentRoute: routeConfig.path,
},
}));
const suggestions = Array.isArray(payload['suggestions'])
? payload['suggestions'] as Array<Record<string, unknown>>
: [];
const viableCount = suggestions.filter((suggestion) => suggestion['viable'] === true).length;
const scopeReady = suggestions.some((suggestion) => suggestion['scopeReady'] === true);
const viabilityStates = suggestions
.map((suggestion) => String(suggestion['viabilityState'] ?? 'unknown'))
.filter((state) => state.length > 0);
return {
config: routeConfig,
supported: viableCount > 0,
scopeReady,
viableCount,
viabilityStates,
payload,
};
}
async function routeLiveUnifiedSearch(
page: Page,
capturedRequests?: Array<Record<string, unknown>>,
@@ -419,26 +603,6 @@ async function rebuildLiveIndexes(baseUrl: string): Promise<void> {
}
}
async function assertLiveSuggestionCoverage(
baseUrl: string,
queries: readonly string[],
): Promise<void> {
const payload = await fetchLiveSuggestionViability(JSON.stringify({
queries,
ambient: {
currentRoute: '/ops/operations/doctor',
},
}));
const suggestions = Array.isArray(payload['suggestions'])
? payload['suggestions'] as Array<Record<string, unknown>>
: [];
const viableSuggestions = suggestions.filter((suggestion) => suggestion['viable'] === true);
if (viableSuggestions.length === 0) {
throw new Error(`Live suggestion preflight returned no viable queries: ${JSON.stringify(payload)}`);
}
}
function safeParseRequest(rawBody: string): Record<string, unknown> {
try {
const parsed = JSON.parse(rawBody) as Record<string, unknown>;
@@ -517,19 +681,42 @@ async function buildCompatibilitySuggestionViability(
: null;
const cardCount = cards.length + overflowCards.length;
const status = String(contextAnswer?.['status'] ?? 'insufficient');
const coverageDomains = Array.isArray(coverage?.['domains'])
? coverage['domains'] as Array<Record<string, unknown>>
: [];
const currentScopeDomain = String(coverage?.['currentScopeDomain'] ?? '');
const currentScopeCoverage = coverageDomains.find((domain) =>
Boolean(domain['isCurrentScope'])
|| String(domain['domain'] ?? '') === currentScopeDomain);
const currentScopeHasVisibleResults =
Boolean(currentScopeCoverage?.['hasVisibleResults'])
|| Number(currentScopeCoverage?.['visibleCardCount'] ?? 0) > 0;
const currentScopeHasCandidates = Number(currentScopeCoverage?.['candidateCount'] ?? 0) > 0;
const scopeReady = currentScopeDomain.length === 0
? cardCount > 0
: currentScopeHasVisibleResults || currentScopeHasCandidates;
const viabilityState = !scopeReady && currentScopeDomain.length > 0
? 'scope_unready'
: status === 'grounded' && cardCount > 0
? 'grounded'
: status === 'clarify'
? 'needs_clarification'
: 'no_match';
const leadingDomain =
String(cards[0]?.['domain'] ?? overflowCards[0]?.['domain'] ?? coverage?.['currentScopeDomain'] ?? '');
suggestions.push({
query,
viable: status === 'grounded' && cardCount > 0,
viable: viabilityState === 'grounded',
status,
code: String(contextAnswer?.['code'] ?? 'no_grounded_evidence'),
cardCount,
leadingDomain: leadingDomain || undefined,
reason: String(contextAnswer?.['reason'] ?? 'No grounded evidence matched the suggestion in the active corpus.'),
viabilityState: status === 'grounded' ? 'grounded' : status === 'clarify' ? 'needs_clarification' : 'no_match',
scopeReady: cardCount > 0,
reason: viabilityState === 'scope_unready'
? `The active route maps to ${currentScopeDomain || 'the current scope'}, but that scope has no ingested search corpus yet.`
: String(contextAnswer?.['reason'] ?? 'No grounded evidence matched the suggestion in the active corpus.'),
viabilityState,
scopeReady,
});
mergedCoverage = mergeCoverage(mergedCoverage, coverage);