578 lines
20 KiB
TypeScript
578 lines
20 KiB
TypeScript
// -----------------------------------------------------------------------------
|
|
// unified-search-doctor.e2e.spec.ts
|
|
// E2E tests for Doctor Check domain queries (180 cases from test-cases.md).
|
|
// Verifies: doctor entity cards, check code rendering, run actions,
|
|
// synthesis templates, severity badges, and domain filtering.
|
|
// -----------------------------------------------------------------------------
|
|
|
|
import { expect, test } from '@playwright/test';
|
|
|
|
import {
|
|
buildResponse,
|
|
doctorCard,
|
|
docsCard,
|
|
mockSearchApi,
|
|
mockSearchApiDynamic,
|
|
setupAuthenticatedSession,
|
|
setupBasicMocks,
|
|
typeInSearch,
|
|
waitForEntityCards,
|
|
waitForResults,
|
|
waitForShell,
|
|
} from './unified-search-fixtures';
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Fixtures: doctor response shapes
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const postgresCheckResponse = buildResponse('database connection failing', [
|
|
doctorCard({
|
|
checkId: 'check.postgres.connectivity',
|
|
title: 'Postgres Connectivity',
|
|
snippet: 'Validates TCP/TLS connectivity to the primary Postgres cluster.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.postgres.pool',
|
|
title: 'Postgres Connection Pool',
|
|
snippet: 'Connection pool utilization is within healthy thresholds.',
|
|
score: 0.72,
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.postgres.migrations',
|
|
title: 'Postgres Migration Status',
|
|
snippet: 'All EF Core migrations have been applied successfully.',
|
|
score: 0.65,
|
|
}),
|
|
], {
|
|
summary: 'Found 3 doctor checks related to database connectivity. Run `stella doctor run check.postgres.*` to diagnose.',
|
|
template: 'doctor_check',
|
|
confidence: 'high',
|
|
sourceCount: 3,
|
|
domainsCovered: ['knowledge'],
|
|
});
|
|
|
|
const timestampingCheckResponse = buildResponse('TSA certificate expiry', [
|
|
doctorCard({
|
|
checkId: 'check.timestamp.tsa.certificate-expiry',
|
|
title: 'TSA Certificate Expiry',
|
|
snippet: 'Checks if the TSA signing certificate is approaching expiration.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.timestamp.tsa.chain-valid',
|
|
title: 'TSA Chain Valid',
|
|
snippet: 'Validates the full certificate chain from leaf to root CA.',
|
|
score: 0.78,
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.timestamp.tsa.root-expiry',
|
|
title: 'TSA Root Expiry',
|
|
snippet: 'Checks if the root CA certificate is approaching expiration.',
|
|
score: 0.70,
|
|
}),
|
|
], {
|
|
summary: 'Found 3 timestamping certificate checks. TSA infrastructure health is critical for eIDAS compliance.',
|
|
template: 'doctor_check',
|
|
confidence: 'high',
|
|
sourceCount: 3,
|
|
domainsCovered: ['knowledge'],
|
|
});
|
|
|
|
const integrationCheckResponse = buildResponse('OCI registry connectivity', [
|
|
doctorCard({
|
|
checkId: 'check.integration.oci.registry',
|
|
title: 'OCI Registry',
|
|
snippet: 'Non-destructive HEAD probe to verify OCI registry availability.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.integration.oci.referrers',
|
|
title: 'OCI Registry Referrers API',
|
|
snippet: 'Checks if the registry supports the OCI Referrers API for artifact linking.',
|
|
score: 0.75,
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.integration.oci.credentials',
|
|
title: 'OCI Registry Credentials',
|
|
snippet: 'Validates stored registry credentials have not expired.',
|
|
score: 0.68,
|
|
}),
|
|
]);
|
|
|
|
const binaryAnalysisCheckResponse = buildResponse('debuginfod available', [
|
|
doctorCard({
|
|
checkId: 'check.binaryanalysis.debuginfod.available',
|
|
title: 'Debuginfod Availability',
|
|
snippet: 'Verifies debuginfod server is reachable for symbol resolution.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.binaryanalysis.buildinfo.cache',
|
|
title: 'Buildinfo Cache',
|
|
snippet: 'Checks buildinfo cache hit rate and staleness.',
|
|
score: 0.72,
|
|
}),
|
|
]);
|
|
|
|
const observabilityCheckResponse = buildResponse('OTLP endpoint check', [
|
|
doctorCard({
|
|
checkId: 'check.telemetry.otlp.endpoint',
|
|
title: 'OTLP Endpoint Check',
|
|
snippet: 'Validates the OpenTelemetry collector endpoint is accepting spans.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.metrics.prometheus.scrape',
|
|
title: 'Prometheus Scrape Check',
|
|
snippet: 'Verifies Prometheus scrape target is healthy and returning metrics.',
|
|
score: 0.70,
|
|
}),
|
|
]);
|
|
|
|
const complianceCheckResponse = buildResponse('audit readiness check', [
|
|
doctorCard({
|
|
checkId: 'check.compliance.audit-readiness',
|
|
title: 'Audit Readiness',
|
|
snippet: 'All evidence chains, attestations, and provenance records are complete for audit.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.compliance.evidence-integrity',
|
|
title: 'Evidence Tamper Check',
|
|
snippet: 'Verifies evidence locker integrity via Merkle tree verification.',
|
|
score: 0.82,
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.compliance.provenance-completeness',
|
|
title: 'Provenance Completeness',
|
|
snippet: 'Checks that all artifacts have complete provenance chains.',
|
|
score: 0.75,
|
|
}),
|
|
]);
|
|
|
|
const agentCheckResponse = buildResponse('agent heartbeat freshness', [
|
|
doctorCard({
|
|
checkId: 'check.agent.heartbeat.freshness',
|
|
title: 'Agent Heartbeat Freshness',
|
|
snippet: 'Checks that all registered agents have reported within the configured interval.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.agent.cluster.quorum',
|
|
title: 'Agent Cluster Quorum',
|
|
snippet: 'Validates that the agent cluster has sufficient members for quorum.',
|
|
score: 0.78,
|
|
}),
|
|
]);
|
|
|
|
const cryptoCheckResponse = buildResponse('FIPS compliance check', [
|
|
doctorCard({
|
|
checkId: 'check.crypto.fips',
|
|
title: 'FIPS 140-2 Compliance',
|
|
snippet: 'Verifies FIPS-validated cryptographic modules are loaded and active.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.crypto.hsm',
|
|
title: 'HSM PKCS#11 Availability',
|
|
snippet: 'Checks HSM module connectivity and signing key availability.',
|
|
score: 0.80,
|
|
}),
|
|
]);
|
|
|
|
const scannerCheckResponse = buildResponse('scanner queue check', [
|
|
doctorCard({
|
|
checkId: 'check.scanner.queue',
|
|
title: 'Scanner Queue Health',
|
|
snippet: 'Scanner job queue depth and processing rate are within normal thresholds.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.scanner.resources',
|
|
title: 'Scanner Resource Utilization',
|
|
snippet: 'CPU and memory usage for scanner workers are within configured limits.',
|
|
score: 0.74,
|
|
}),
|
|
]);
|
|
|
|
const releaseCheckResponse = buildResponse('promotion gates check', [
|
|
doctorCard({
|
|
checkId: 'check.release.promotion.gates',
|
|
title: 'Promotion Gate Health',
|
|
snippet: 'All promotion gates are correctly configured and responding.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.release.rollback.readiness',
|
|
title: 'Rollback Readiness',
|
|
snippet: 'Rollback procedures are verified and ready for execution.',
|
|
score: 0.72,
|
|
}),
|
|
]);
|
|
|
|
const vexCheckResponse = buildResponse('VEX schema compliance check', [
|
|
doctorCard({
|
|
checkId: 'check.vex.schema',
|
|
title: 'VEX Schema Compliance',
|
|
snippet: 'Validates all stored VEX documents conform to their declared schema.',
|
|
}),
|
|
doctorCard({
|
|
checkId: 'check.vex.issuer-trust',
|
|
title: 'VEX Issuer Trust',
|
|
snippet: 'Checks that VEX issuer trust tiers are configured and up to date.',
|
|
score: 0.73,
|
|
}),
|
|
]);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test suites
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test.describe('Unified Search — Doctor Domain', () => {
|
|
test.beforeEach(async ({ page }) => {
|
|
await setupBasicMocks(page);
|
|
await setupAuthenticatedSession(page);
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.1 Database & Infrastructure Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Database & Infrastructure Checks', () => {
|
|
const queries = [
|
|
'check.postgres.connectivity',
|
|
'database connection failing',
|
|
'postgres migrations pending',
|
|
'connection pool exhausted',
|
|
'disk space running low',
|
|
'evidence locker write check',
|
|
'backup directory writable',
|
|
];
|
|
|
|
for (const query of queries) {
|
|
test(`"${query}" returns doctor entity cards`, async ({ page }) => {
|
|
await mockSearchApi(page, postgresCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
|
|
const cards = await waitForEntityCards(page);
|
|
const count = await cards.count();
|
|
expect(count).toBeGreaterThanOrEqual(1);
|
|
|
|
// At least one card should have doctor-related content
|
|
const allText = await page.locator('.search__results').textContent();
|
|
expect(allText).toBeTruthy();
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.2 Security & Auth Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Security & Auth Checks', () => {
|
|
const queries = [
|
|
'authentication config check',
|
|
'OIDC provider connectivity',
|
|
'signing key health',
|
|
'token service health',
|
|
'certificate chain validation',
|
|
'FIPS compliance check',
|
|
'HSM availability check',
|
|
'eIDAS compliance check',
|
|
'GOST availability check',
|
|
];
|
|
|
|
for (const query of queries) {
|
|
test(`"${query}" returns doctor cards`, async ({ page }) => {
|
|
const fixture = query.includes('FIPS') || query.includes('HSM') || query.includes('eIDAS') || query.includes('GOST')
|
|
? cryptoCheckResponse
|
|
: postgresCheckResponse;
|
|
await mockSearchApi(page, fixture);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.3 Compliance, Agent & Notification Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Compliance, Agent & Notification Checks', () => {
|
|
const complianceQueries = [
|
|
'audit readiness check',
|
|
'evidence integrity check',
|
|
'provenance completeness',
|
|
'attestation signing health',
|
|
'evidence generation rate',
|
|
'export readiness check',
|
|
'compliance framework check',
|
|
];
|
|
|
|
for (const query of complianceQueries) {
|
|
test(`compliance: "${query}"`, async ({ page }) => {
|
|
await mockSearchApi(page, complianceCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
|
|
const agentQueries = [
|
|
'agent heartbeat freshness',
|
|
'agent capacity check',
|
|
'stale agent detection',
|
|
'agent cluster health',
|
|
'agent cluster quorum',
|
|
'agent version consistency',
|
|
'agent certificate expiry',
|
|
'agent task backlog',
|
|
];
|
|
|
|
for (const query of agentQueries) {
|
|
test(`agent: "${query}"`, async ({ page }) => {
|
|
await mockSearchApi(page, agentCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
|
|
const notifyQueries = [
|
|
'email notification check',
|
|
'Slack connectivity check',
|
|
'Teams notification check',
|
|
'notification queue health',
|
|
'webhook connectivity',
|
|
];
|
|
|
|
for (const query of notifyQueries) {
|
|
test(`notify: "${query}"`, async ({ page }) => {
|
|
await mockSearchApi(page, integrationCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.4 Environment & Release Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Environment & Release Checks', () => {
|
|
const queries = [
|
|
'environment connectivity',
|
|
'environment drift',
|
|
'network policy enforcement',
|
|
'deployment health check',
|
|
'active release health',
|
|
'promotion gates check',
|
|
'rollback readiness',
|
|
'release schedule check',
|
|
];
|
|
|
|
for (const query of queries) {
|
|
test(`"${query}" returns cards`, async ({ page }) => {
|
|
await mockSearchApi(page, releaseCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.5 Timestamping & Certificate Lifecycle Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Timestamping & Certificate Lifecycle Checks', () => {
|
|
const queries = [
|
|
'TSA availability check',
|
|
'TSA response time',
|
|
'TSA valid response check',
|
|
'TSA failover ready',
|
|
'TSA certificate expiry',
|
|
'TSA root expiry check',
|
|
'TSA chain validation',
|
|
'OCSP responder check',
|
|
'CRL distribution check',
|
|
'revocation cache freshness',
|
|
'OCSP stapling enabled',
|
|
'evidence staleness check',
|
|
'timestamp approaching expiry',
|
|
'TST algorithm deprecated',
|
|
'retimestamp pending',
|
|
'EU trust list freshness',
|
|
'QTS providers qualified',
|
|
'system time synced',
|
|
];
|
|
|
|
for (const query of queries) {
|
|
test(`"${query}" returns timestamping doctor cards`, async ({ page }) => {
|
|
await mockSearchApi(page, timestampingCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.6 Integration & External Connectivity Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Integration & External Connectivity Checks', () => {
|
|
const queries = [
|
|
'OCI registry connectivity',
|
|
'OCI referrers API check',
|
|
'OCI push authorization',
|
|
'OCI pull authorization',
|
|
'OCI registry credentials',
|
|
'S3 object storage check',
|
|
'SMTP connectivity check',
|
|
'Slack webhook check',
|
|
'Teams webhook check',
|
|
'Git provider connectivity',
|
|
'LDAP connectivity check',
|
|
'CI system connectivity',
|
|
'secrets manager connectivity',
|
|
'integration webhook health',
|
|
'cannot push policy to OCI',
|
|
'Git provider auth failing',
|
|
'object storage write failing',
|
|
'secrets vault unreachable',
|
|
];
|
|
|
|
for (const query of queries) {
|
|
test(`"${query}" returns integration doctor cards`, async ({ page }) => {
|
|
await mockSearchApi(page, integrationCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.7 Binary Analysis & Corpus Health Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Binary Analysis & Corpus Health Checks', () => {
|
|
const queries = [
|
|
'debuginfod available',
|
|
'ddeb repo enabled',
|
|
'buildinfo cache health',
|
|
'symbol recovery fallback',
|
|
'corpus mirror freshness',
|
|
'corpus KPI baseline exists',
|
|
'binary analysis not working',
|
|
'symbol table missing',
|
|
'debug symbols not found',
|
|
'buildinfo cache expired',
|
|
'Go binary stripped no debug',
|
|
'PE authenticode verification failed',
|
|
'corpus mirror out of date',
|
|
];
|
|
|
|
for (const query of queries) {
|
|
test(`"${query}" returns binary analysis doctor cards`, async ({ page }) => {
|
|
await mockSearchApi(page, binaryAnalysisCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.8 Observability, Logging & Operations Deep Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Observability & Operations Checks', () => {
|
|
const queries = [
|
|
'OTLP exporter not sending',
|
|
'log directory not writable',
|
|
'log rotation not configured',
|
|
'Prometheus not scraping metrics',
|
|
'dead letter queue growing',
|
|
'job queue backlog increasing',
|
|
'scheduler not processing',
|
|
'traces not appearing in Jaeger',
|
|
'metrics endpoint 404',
|
|
'OpenTelemetry collector down',
|
|
'dead letter messages accumulating',
|
|
];
|
|
|
|
for (const query of queries) {
|
|
test(`"${query}" returns observability doctor cards`, async ({ page }) => {
|
|
await mockSearchApi(page, observabilityCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// 3.9 Scanner, Reachability & Storage Deep Checks
|
|
// -------------------------------------------------------------------------
|
|
test.describe('Scanner & Storage Deep Checks', () => {
|
|
const queries = [
|
|
'scanner queue backed up',
|
|
'SBOM generation failing',
|
|
'vulnerability scan timing out',
|
|
'witness graph corruption',
|
|
'slice cache miss rate high',
|
|
'reachability computation stalled',
|
|
'scanner resource utilization high',
|
|
'disk space critical on evidence locker',
|
|
'evidence locker write failure',
|
|
'postgres connection pool exhausted',
|
|
'database migrations not applied',
|
|
];
|
|
|
|
for (const query of queries) {
|
|
test(`"${query}" returns scanner/storage doctor cards`, async ({ page }) => {
|
|
await mockSearchApi(page, scannerCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, query);
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
});
|
|
}
|
|
});
|
|
|
|
// -------------------------------------------------------------------------
|
|
// Doctor-specific rendering verification
|
|
// -------------------------------------------------------------------------
|
|
test('doctor cards show "Run Check" action button', async ({ page }) => {
|
|
await mockSearchApi(page, postgresCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, 'database connection');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page, 3);
|
|
|
|
const allText = await page.locator('.search__results').textContent();
|
|
expect(allText).toContain('Run Check');
|
|
});
|
|
|
|
test('doctor synthesis uses doctor_check template', async ({ page }) => {
|
|
await mockSearchApi(page, postgresCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, 'postgres connectivity');
|
|
await waitForResults(page);
|
|
|
|
const synthesis = page.locator('app-synthesis-panel');
|
|
await expect(synthesis).toBeVisible({ timeout: 10_000 });
|
|
const text = await synthesis.textContent();
|
|
expect(text).toContain('stella doctor run');
|
|
});
|
|
|
|
test('VEX doctor checks return VEX-specific cards', async ({ page }) => {
|
|
await mockSearchApi(page, vexCheckResponse);
|
|
await waitForShell(page);
|
|
await typeInSearch(page, 'VEX schema compliance');
|
|
await waitForResults(page);
|
|
await waitForEntityCards(page);
|
|
|
|
const allText = await page.locator('.search__results').textContent();
|
|
expect(allText).toContain('VEX Schema');
|
|
});
|
|
});
|