stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

Repair live jobs queues action handoffs

Browse Source
This commit is contained in:
master
2026-03-10 20:46:55 +02:00
parent f727ec24fd
commit 3865b93091
6 changed files with 934 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
src/Web/StellaOps.Web/scripts/live-jobs-queues-action-sweep.mjs Normal file
View File

@@ -0,0 +1,287 @@
#!/usr/bin/env node
import { mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { chromium } from 'playwright';
import { authenticateFrontdoor, createAuthenticatedContext } from './live-frontdoor-auth.mjs';
const webRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const outputDir = path.join(webRoot, 'output', 'playwright');
const outputPath = path.join(outputDir, 'live-jobs-queues-action-sweep.json');
const authStatePath = path.join(outputDir, 'live-jobs-queues-action-sweep.state.json');
const authReportPath = path.join(outputDir, 'live-jobs-queues-action-sweep.auth.json');
const scopeQuery = 'tenant=demo-prod&regions=us-east&environments=stage&timeWindow=7d';
const routePath = '/ops/operations/jobs-queues';
function scopedUrl(route = routePath) {
const separator = route.includes('?') ? '&' : '?';
return `https://stella-ops.local${route}${separator}${scopeQuery}`;
}
async function settle(page) {
await page.waitForLoadState('domcontentloaded', { timeout: 20_000 }).catch(() => {});
await page.waitForTimeout(1_200);
}
async function gotoPage(page) {
await page.goto(scopedUrl(), { waitUntil: 'domcontentloaded', timeout: 30_000 });
await settle(page);
}
async function clickTab(page, tabName) {
await page.getByRole('button', { name: tabName }).click({ timeout: 10_000 });
await settle(page);
}
async function rowCount(page) {
return page.locator('tbody tr').count();
}
function searchFilter(page) {
return page.locator('section.filters input[type="search"]').first();
}
function filterSelect(page, index) {
return page.locator('section.filters select').nth(index);
}
function actionBanner(page) {
return page.locator('section.jobs-queues .jobs-queues__banner[role="status"]').first();
}
async function captureRuntimeIssues(page) {
const alerts = await page
.locator('[role="alert"], .error-banner, .toast, .notification')
.evaluateAll((nodes) =>
nodes
.map((node) => (node.textContent || '').trim().replace(/\s+/g, ' '))
.filter(Boolean),
)
.catch(() => []);
return alerts;
}
async function readActionHrefs(page) {
return page.locator('tbody tr:first-child .actions a').evaluateAll((links) =>
links.map((link) => ({
label: (link.textContent || '').trim().replace(/\s+/g, ' '),
href: link.getAttribute('href') || '',
})),
);
}
async function assert(condition, message, details = {}) {
if (!condition) {
throw new Error(`${message}${Object.keys(details).length ? ` ${JSON.stringify(details)}` : ''}`);
}
}
async function run() {
await mkdir(outputDir, { recursive: true });
const authReport = await authenticateFrontdoor({
statePath: authStatePath,
reportPath: authReportPath,
headless: true,
});
const browser = await chromium.launch({
headless: true,
args: ['--disable-dev-shm-usage'],
});
const context = await createAuthenticatedContext(browser, authReport, { statePath: authStatePath });
const page = await context.newPage();
const consoleErrors = [];
const responseErrors = [];
const requestFailures = [];
page.on('console', (message) => {
if (message.type() === 'error') {
consoleErrors.push(message.text());
}
});
page.on('response', (response) => {
const url = response.url();
if (/\.(?:css|js|map|png|jpg|jpeg|svg|woff2?)(?:$|\?)/i.test(url)) {
return;
}
if (response.status() >= 400 && !url.includes('/connect/authorize')) {
responseErrors.push({
status: response.status(),
method: response.request().method(),
url,
});
}
});
page.on('requestfailed', (request) => {
const url = request.url();
const failure = request.failure()?.errorText ?? 'unknown';
if (failure === 'net::ERR_ABORTED') {
return;
}
if (/\.(?:css|js|map|png|jpg|jpeg|svg|woff2?)(?:$|\?)/i.test(url)) {
return;
}
requestFailures.push({
method: request.method(),
url,
error: failure,
});
});
const checks = [];
try {
await gotoPage(page);
await assert((await page.locator('h1').first().textContent())?.trim() === 'Jobs & Queues', 'Unexpected page heading.');
let actions = await readActionHrefs(page);
await assert(actions.length === 2, 'Jobs tab did not expose the expected two row actions.', { actions });
await assert(actions[0].href.includes('/ops/operations/jobengine'), 'Jobs primary action did not target JobEngine.', { actions });
await assert(actions[1].href.includes('/ops/operations/jobengine/jobs'), 'Jobs secondary action did not target the Job list.', { actions });
await assert(!actions.some((entry) => entry.href.includes('/ops/operations/jobs-queues')), 'Jobs tab still contains self-linking row actions.', { actions });
checks.push({ name: 'jobs-row-actions', ok: true, actions });
await searchFilter(page).fill('Vulnerability');
await settle(page);
let rows = await rowCount(page);
await assert(rows === 1, 'Jobs search filter did not reduce the table to one row.', { rows });
await page.getByRole('button', { name: 'Clear filters' }).click();
await settle(page);
rows = await rowCount(page);
await assert(rows === 4, 'Jobs clear filters did not restore all rows.', { rows });
checks.push({ name: 'jobs-filters', ok: true, rowsAfterClear: rows });
await searchFilter(page).fill('Vulnerability');
await settle(page);
await clickTab(page, 'Runs');
await assert(await searchFilter(page).inputValue() === '', 'Tab switch did not clear the search filter.');
checks.push({ name: 'tab-switch-clears-filters', ok: true });
await filterSelect(page, 0).selectOption('FAILED');
await filterSelect(page, 1).selectOption('DEGRADED');
await settle(page);
rows = await rowCount(page);
await assert(rows === 1, 'Runs filters did not isolate the failed degraded row.', { rows });
checks.push({ name: 'runs-filters', ok: true, rows });
await page.getByRole('button', { name: 'Clear filters' }).click();
await settle(page);
await page.getByRole('button', { name: 'Copy CorrID' }).first().click();
await settle(page);
const statusText = ((await actionBanner(page).textContent()) || '').trim();
await assert(statusText.includes('corr-run-001'), 'Runs copy action did not produce inline feedback.', { statusText });
checks.push({ name: 'runs-copy-feedback', ok: true, statusText });
await filterSelect(page, 0).selectOption('DEAD-LETTER');
await filterSelect(page, 1).selectOption('BLOCKING');
await settle(page);
await page.getByRole('link', { name: 'Open Dead-Letter Queue' }).click();
await settle(page);
await assert(page.url().includes('/ops/operations/dead-letter/queue'), 'Runs dead-letter handoff did not reach the DLQ.');
checks.push({ name: 'runs-dead-letter-handoff', ok: true, targetUrl: page.url() });
await gotoPage(page);
await clickTab(page, 'Schedules');
await filterSelect(page, 0).selectOption('FAIL');
await filterSelect(page, 1).selectOption('Daily');
await settle(page);
rows = await rowCount(page);
await assert(rows === 1, 'Schedules filters did not isolate the failed daily schedule.', { rows });
await page.getByRole('link', { name: 'Review Dead-Letter Queue' }).click();
await settle(page);
await assert(page.url().includes('/ops/operations/dead-letter/queue'), 'Schedules failure handoff did not reach the DLQ.');
checks.push({ name: 'schedules-failure-handoff', ok: true, targetUrl: page.url() });
await gotoPage(page);
await clickTab(page, 'Schedules');
await page.getByRole('link', { name: 'Manage Schedules' }).first().click();
await settle(page);
await assert(page.url().includes('/ops/operations/scheduler/schedules'), 'Schedules primary handoff did not reach Scheduler.');
checks.push({ name: 'schedules-primary-handoff', ok: true, targetUrl: page.url() });
await gotoPage(page);
await clickTab(page, 'Dead Letters');
actions = await readActionHrefs(page);
await assert(actions[0].href.includes('/ops/operations/dead-letter/queue'), 'Dead-letter primary action did not target the queue.', { actions });
await assert(actions[1].href.includes('/ops/operations/data-integrity/dlq'), 'Dead-letter recovery action did not target the DLQ recovery surface.', { actions });
await page.getByRole('link', { name: 'Open Replay Recovery' }).first().click();
await settle(page);
await assert(page.url().includes('/ops/operations/data-integrity/dlq'), 'Dead-letter recovery handoff did not reach Data Integrity.');
checks.push({ name: 'dead-letter-handoffs', ok: true, targetUrl: page.url() });
await gotoPage(page);
await clickTab(page, 'Workers');
await filterSelect(page, 0).selectOption('DEGRADED');
await filterSelect(page, 1).selectOption('feeds');
await settle(page);
rows = await rowCount(page);
await assert(rows === 1, 'Workers filters did not isolate the degraded feeds worker.', { rows });
await page.getByRole('link', { name: 'Open Worker Fleet' }).click();
await settle(page);
await assert(page.url().includes('/ops/operations/scheduler/workers'), 'Workers primary handoff did not reach Worker Fleet.');
checks.push({ name: 'workers-primary-handoff', ok: true, targetUrl: page.url() });
await gotoPage(page);
await clickTab(page, 'Workers');
await filterSelect(page, 0).selectOption('DEGRADED');
await filterSelect(page, 1).selectOption('feeds');
await settle(page);
await page.getByRole('link', { name: 'Inspect Scheduler Runs' }).click();
await settle(page);
await assert(page.url().includes('/ops/operations/scheduler/runs'), 'Workers secondary handoff did not reach Scheduler Runs.');
checks.push({ name: 'workers-secondary-handoff', ok: true, targetUrl: page.url() });
const runtimeIssues = {
consoleErrors,
responseErrors,
requestFailures,
alerts: await captureRuntimeIssues(page),
};
const summary = {
checkedAtUtc: new Date().toISOString(),
routePath,
checks,
runtimeIssues,
failedCheckCount: checks.filter((check) => !check.ok).length,
runtimeIssueCount:
runtimeIssues.consoleErrors.length
+ runtimeIssues.responseErrors.length
+ runtimeIssues.requestFailures.length
+ runtimeIssues.alerts.length,
};
await writeFile(outputPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
if (summary.failedCheckCount > 0 || summary.runtimeIssueCount > 0) {
process.exitCode = 1;
}
} catch (error) {
const summary = {
checkedAtUtc: new Date().toISOString(),
routePath,
checks,
error: error instanceof Error ? error.message : String(error),
consoleErrors,
responseErrors,
requestFailures,
};
await writeFile(outputPath, `${JSON.stringify(summary, null, 2)}\n`, 'utf8');
process.exitCode = 1;
} finally {
await context.close().catch(() => {});
await browser.close().catch(() => {});
}
}
await run();

117
src/Web/StellaOps.Web/src/app/features/platform/ops/platform-jobs-queues-page.component.spec.ts Normal file
View File

@@ -0,0 +1,117 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { provideRouter } from '@angular/router';
import { PlatformJobsQueuesPageComponent } from './platform-jobs-queues-page.component';
describe('PlatformJobsQueuesPageComponent', () => {
let fixture: ComponentFixture<PlatformJobsQueuesPageComponent>;
let component: PlatformJobsQueuesPageComponent;
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [PlatformJobsQueuesPageComponent],
providers: [provideRouter([])],
}).compileComponents();
fixture = TestBed.createComponent(PlatformJobsQueuesPageComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('renders truthful jobs handoff actions instead of self-links', () => {
const actions = Array.from(
fixture.nativeElement.querySelectorAll('tbody tr:first-child .actions a') as NodeListOf<HTMLAnchorElement>,
);
const labels = actions.map((link) => link.textContent?.trim());
const hrefs = actions.map((link) => link.getAttribute('href') ?? '');
expect(labels).toEqual(['Open JobEngine', 'View Jobs']);
expect(hrefs[0]).toContain('/ops/operations/jobengine');
expect(hrefs[1]).toContain('/ops/operations/jobengine/jobs');
expect(hrefs).not.toContain('/ops/operations/jobs-queues');
});
it('filters the jobs table and clears stale filters when switching tabs', async () => {
const searchInput: HTMLInputElement = fixture.nativeElement.querySelector('input[type="search"]');
searchInput.value = 'Vulnerability';
searchInput.dispatchEvent(new Event('input'));
await fixture.whenStable();
fixture.detectChanges();
let rows = fixture.nativeElement.querySelectorAll('tbody tr');
expect(rows.length).toBe(1);
expect(rows[0].textContent).toContain('Vulnerability sync (NVD)');
const runsTab: HTMLButtonElement = fixture.nativeElement.querySelectorAll('.tabs button')[1];
runsTab.click();
await fixture.whenStable();
fixture.detectChanges();
expect(component.tab()).toBe('runs');
expect(component.searchQuery()).toBe('');
expect(component.statusFilter()).toBe('');
expect(component.facetFilter()).toBe('');
rows = fixture.nativeElement.querySelectorAll('tbody tr');
expect(rows.length).toBe(4);
});
it('applies run filters using the active tab facets', async () => {
component.setTab('runs');
fixture.detectChanges();
const selects = fixture.nativeElement.querySelectorAll('select');
const statusSelect = selects[0] as HTMLSelectElement;
const impactSelect = selects[1] as HTMLSelectElement;
statusSelect.value = 'FAILED';
statusSelect.dispatchEvent(new Event('change'));
impactSelect.value = 'DEGRADED';
impactSelect.dispatchEvent(new Event('change'));
await fixture.whenStable();
fixture.detectChanges();
const rows = fixture.nativeElement.querySelectorAll('tbody tr');
expect(rows.length).toBe(1);
expect(rows[0].textContent).toContain('run-003');
expect(rows[0].textContent).toContain('FAILED');
});
it('shows inline copy feedback for correlation actions', async () => {
const clipboard = {
writeText: jasmine.createSpy('writeText').and.returnValue(Promise.resolve()),
};
Object.defineProperty(navigator, 'clipboard', {
configurable: true,
value: clipboard,
});
component.setTab('runs');
fixture.detectChanges();
const copyButton = fixture.nativeElement.querySelector('.actions button') as HTMLButtonElement;
copyButton.click();
await fixture.whenStable();
fixture.detectChanges();
expect(clipboard.writeText).toHaveBeenCalledWith('corr-run-001');
const status = fixture.nativeElement.querySelector('[role="status"]');
expect(status?.textContent).toContain('corr-run-001');
});
it('routes dead-letter handoffs into canonical recovery pages', () => {
component.setTab('dead-letters');
fixture.detectChanges();
const actions = Array.from(
fixture.nativeElement.querySelectorAll('tbody tr:first-child .actions a') as NodeListOf<HTMLAnchorElement>,
);
const labels = actions.map((link) => link.textContent?.trim());
const hrefs = actions.map((link) => link.getAttribute('href') ?? '');
expect(labels).toEqual(['Open Dead-Letter Queue', 'Open Replay Recovery']);
expect(hrefs[0]).toContain('/ops/operations/dead-letter/queue');
expect(hrefs[1]).toContain('/ops/operations/data-integrity/dlq');
});
});

574
src/Web/StellaOps.Web/src/app/features/platform/ops/platform-jobs-queues-page.component.ts
View File

@@ -1,58 +1,68 @@
import { ChangeDetectionStrategy, Component, signal } from '@angular/core';
import { ChangeDetectionStrategy, Component, computed, signal } from '@angular/core';
import { FormsModule } from '@angular/forms';
import { RouterLink } from '@angular/router';
import { OPERATIONS_PATHS, dataIntegrityPath, deadLetterQueuePath } from './operations-paths';
type JobsQueuesTab = 'jobs' | 'runs' | 'schedules' | 'dead-letters' | 'workers';
type JobImpact = 'BLOCKING' | 'DEGRADED' | 'INFO';
type Cadence = 'Hourly' | 'Daily';
interface TabAction {
readonly label: string;
readonly route: string;
}
interface JobDefinitionRow {
id: string;
name: string;
type: string;
lastRun: string;
health: 'OK' | 'WARN' | 'DLQ';
readonly id: string;
readonly name: string;
readonly type: string;
readonly lastRun: string;
readonly health: 'OK' | 'WARN' | 'DLQ';
}
interface JobRunRow {
id: string;
job: string;
status: 'RUNNING' | 'COMPLETED' | 'FAILED' | 'DEAD-LETTER';
startedAt: string;
duration: string;
impact: JobImpact;
correlationId: string;
readonly id: string;
readonly job: string;
readonly status: 'RUNNING' | 'COMPLETED' | 'FAILED' | 'DEAD-LETTER';
readonly startedAt: string;
readonly duration: string;
readonly impact: JobImpact;
readonly correlationId: string;
}
interface ScheduleRow {
id: string;
name: string;
cron: string;
nextRun: string;
lastStatus: 'OK' | 'WARN' | 'FAIL';
readonly id: string;
readonly name: string;
readonly cron: string;
readonly nextRun: string;
readonly lastStatus: 'OK' | 'WARN' | 'FAIL';
readonly cadence: Cadence;
}
interface DeadLetterRow {
id: string;
timestamp: string;
job: string;
error: string;
retryable: 'YES' | 'NO';
impact: JobImpact;
correlationId: string;
readonly id: string;
readonly timestamp: string;
readonly job: string;
readonly error: string;
readonly retryable: 'YES' | 'NO';
readonly impact: JobImpact;
readonly correlationId: string;
}
interface WorkerRow {
id: string;
name: string;
queue: string;
state: 'HEALTHY' | 'DEGRADED';
capacity: string;
heartbeat: string;
readonly id: string;
readonly name: string;
readonly queue: string;
readonly state: 'HEALTHY' | 'DEGRADED';
readonly capacity: string;
readonly heartbeat: string;
}
@Component({
selector: 'app-platform-jobs-queues-page',
standalone: true,
imports: [RouterLink],
imports: [FormsModule, RouterLink],
changeDetection: ChangeDetectionStrategy.OnPush,
template: `
<section class="jobs-queues">
@@ -60,22 +70,29 @@ interface WorkerRow {
<div>
<h1>Jobs & Queues</h1>
<p>
Unified operator surface for orchestrator jobs, scheduler runs, schedules,
dead letters, and worker fleet posture.
Truthful execution overview for queue posture, scheduler health, dead-letter recovery,
and worker capacity. Execution control continues in the canonical JobEngine, Scheduler,
Dead-Letter, and Data Integrity surfaces linked below.
</p>
</div>
<div class="jobs-queues__actions">
<a routerLink="/ops/operations/data-integrity">Open Data Integrity</a>
<a routerLink="/ops/operations/doctor">Run Diagnostics</a>
<a [routerLink]="OPERATIONS_PATHS.dataIntegrity">Open Data Integrity</a>
<a [routerLink]="OPERATIONS_PATHS.doctor">Open Diagnostics</a>
</div>
</header>
@if (actionNotice()) {
<div class="jobs-queues__banner jobs-queues__banner--info" role="status">
{{ actionNotice() }}
</div>
}
<nav class="tabs" aria-label="Jobs and queues tabs">
<button type="button" [class.active]="tab() === 'jobs'" (click)="tab.set('jobs')">Jobs</button>
<button type="button" [class.active]="tab() === 'runs'" (click)="tab.set('runs')">Runs</button>
<button type="button" [class.active]="tab() === 'schedules'" (click)="tab.set('schedules')">Schedules</button>
<button type="button" [class.active]="tab() === 'dead-letters'" (click)="tab.set('dead-letters')">Dead Letters</button>
<button type="button" [class.active]="tab() === 'workers'" (click)="tab.set('workers')">Workers</button>
<button type="button" [class.active]="tab() === 'jobs'" (click)="setTab('jobs')">Jobs</button>
<button type="button" [class.active]="tab() === 'runs'" (click)="setTab('runs')">Runs</button>
<button type="button" [class.active]="tab() === 'schedules'" (click)="setTab('schedules')">Schedules</button>
<button type="button" [class.active]="tab() === 'dead-letters'" (click)="setTab('dead-letters')">Dead Letters</button>
<button type="button" [class.active]="tab() === 'workers'" (click)="setTab('workers')">Workers</button>
</nav>
<section class="kpis" aria-label="Queue summary">
@@ -86,31 +103,37 @@ interface WorkerRow {
<span>Workers {{ workers.length }}</span>
</section>
<section class="filters">
<section class="filters" aria-label="Jobs and queues filters">
<label>
Search
<input type="search" placeholder="Job id, run id, correlation id" />
<input
type="search"
[ngModel]="searchQuery()"
(ngModelChange)="searchQuery.set($event)"
[placeholder]="searchPlaceholder()"
/>
</label>
<label>
Status
<select>
<option>All</option>
<option>RUNNING</option>
<option>FAILED</option>
<option>DEAD-LETTER</option>
<option>COMPLETED</option>
{{ statusLabel() }}
<select [ngModel]="statusFilter()" (ngModelChange)="statusFilter.set($event)">
<option value="">All {{ statusLabel().toLowerCase() }}</option>
@for (option of statusOptions(); track option) {
<option [value]="option">{{ option }}</option>
}
</select>
</label>
<label>
Type
<select>
<option>All</option>
<option>security</option>
<option>supply</option>
<option>evidence</option>
<option>feeds</option>
{{ facetLabel() }}
<select [ngModel]="facetFilter()" (ngModelChange)="facetFilter.set($event)">
<option value="">All {{ facetLabel().toLowerCase() }}</option>
@for (option of facetOptions(); track option) {
<option [value]="option">{{ option }}</option>
}
</select>
</label>
<button type="button" class="filters__clear" (click)="clearFilters()" [disabled]="!hasActiveFilters()">
Clear filters
</button>
</section>
@if (tab() === 'jobs') {
@@ -126,16 +149,23 @@ interface WorkerRow {
</tr>
</thead>
<tbody>
@for (row of jobs; track row.id) {
@if (filteredJobs().length) {
@for (row of filteredJobs(); track row.id) {
<tr>
<td>{{ row.name }}</td>
<td>{{ row.type }}</td>
<td>{{ row.lastRun }}</td>
<td><span class="pill" [class]="'pill pill--' + row.health.toLowerCase()">{{ row.health }}</span></td>
<td class="actions">
@for (action of jobActions(row); track action.label) {
<a [routerLink]="action.route">{{ action.label }}</a>
}
</td>
</tr>
}
} @else {
<tr>
<td>{{ row.name }}</td>
<td>{{ row.type }}</td>
<td>{{ row.lastRun }}</td>
<td><span class="pill" [class]="'pill pill--' + row.health.toLowerCase()">{{ row.health }}</span></td>
<td class="actions">
<a routerLink="/ops/operations/jobs-queues">View</a>
<a routerLink="/ops/operations/jobs-queues">Run Now</a>
</td>
<td class="table-empty" colspan="5">No jobs match the current filters.</td>
</tr>
}
</tbody>
@@ -158,18 +188,26 @@ interface WorkerRow {
</tr>
</thead>
<tbody>
@for (row of runs; track row.id) {
@if (filteredRuns().length) {
@for (row of filteredRuns(); track row.id) {
<tr>
<td>{{ row.id }}</td>
<td>{{ row.job }}</td>
<td><span class="pill" [class]="'pill pill--' + row.status.toLowerCase().replace('-', '')">{{ row.status }}</span></td>
<td>{{ row.startedAt }}</td>
<td>{{ row.duration }}</td>
<td><span class="impact" [class]="'impact impact--' + row.impact.toLowerCase()">{{ row.impact }}</span></td>
<td class="actions">
@for (action of runActions(row); track action.label) {
<a [routerLink]="action.route">{{ action.label }}</a>
}
<button type="button" (click)="copyCorrelationId(row.correlationId)">Copy CorrID</button>
</td>
</tr>
}
} @else {
<tr>
<td>{{ row.id }}</td>
<td>{{ row.job }}</td>
<td><span class="pill" [class]="'pill pill--' + row.status.toLowerCase().replace('-', '')">{{ row.status }}</span></td>
<td>{{ row.startedAt }}</td>
<td>{{ row.duration }}</td>
<td><span class="impact" [class]="'impact impact--' + row.impact.toLowerCase()">{{ row.impact }}</span></td>
<td class="actions">
<a routerLink="/ops/operations/jobs-queues">View</a>
<button type="button" (click)="copyCorrelationId(row.correlationId)">Copy CorrID</button>
</td>
<td class="table-empty" colspan="7">No runs match the current filters.</td>
</tr>
}
</tbody>
@@ -190,16 +228,23 @@ interface WorkerRow {
</tr>
</thead>
<tbody>
@for (row of schedules; track row.id) {
@if (filteredSchedules().length) {
@for (row of filteredSchedules(); track row.id) {
<tr>
<td>{{ row.name }}</td>
<td>{{ row.cron }}</td>
<td>{{ row.nextRun }}</td>
<td><span class="pill" [class]="'pill pill--' + row.lastStatus.toLowerCase()">{{ row.lastStatus }}</span></td>
<td class="actions">
@for (action of scheduleActions(row); track action.label) {
<a [routerLink]="action.route">{{ action.label }}</a>
}
</td>
</tr>
}
} @else {
<tr>
<td>{{ row.name }}</td>
<td>{{ row.cron }}</td>
<td>{{ row.nextRun }}</td>
<td><span class="pill" [class]="'pill pill--' + row.lastStatus.toLowerCase()">{{ row.lastStatus }}</span></td>
<td class="actions">
<a routerLink="/ops/operations/jobs-queues">Edit</a>
<a routerLink="/ops/operations/jobs-queues">Pause</a>
</td>
<td class="table-empty" colspan="5">No schedules match the current filters.</td>
</tr>
}
</tbody>
@@ -221,17 +266,25 @@ interface WorkerRow {
</tr>
</thead>
<tbody>
@for (row of deadLetters; track row.id) {
@if (filteredDeadLetters().length) {
@for (row of filteredDeadLetters(); track row.id) {
<tr>
<td>{{ row.timestamp }}</td>
<td>{{ row.job }}</td>
<td>{{ row.error }}</td>
<td>{{ row.retryable }}</td>
<td><span class="impact" [class]="'impact impact--' + row.impact.toLowerCase()">{{ row.impact }}</span></td>
<td class="actions">
@for (action of deadLetterActions(row); track action.label) {
<a [routerLink]="action.route">{{ action.label }}</a>
}
<button type="button" (click)="copyCorrelationId(row.correlationId)">Copy CorrID</button>
</td>
</tr>
}
} @else {
<tr>
<td>{{ row.timestamp }}</td>
<td>{{ row.job }}</td>
<td>{{ row.error }}</td>
<td>{{ row.retryable }}</td>
<td><span class="impact" [class]="'impact impact--' + row.impact.toLowerCase()">{{ row.impact }}</span></td>
<td class="actions">
<a routerLink="/ops/operations/jobs-queues">Replay</a>
<button type="button" (click)="copyCorrelationId(row.correlationId)">Copy CorrID</button>
</td>
<td class="table-empty" colspan="6">No dead-letter entries match the current filters.</td>
</tr>
}
</tbody>
@@ -253,17 +306,24 @@ interface WorkerRow {
</tr>
</thead>
<tbody>
@for (row of workers; track row.id) {
@if (filteredWorkers().length) {
@for (row of filteredWorkers(); track row.id) {
<tr>
<td>{{ row.name }}</td>
<td>{{ row.queue }}</td>
<td><span class="pill" [class]="'pill pill--' + row.state.toLowerCase()">{{ row.state }}</span></td>
<td>{{ row.capacity }}</td>
<td>{{ row.heartbeat }}</td>
<td class="actions">
@for (action of workerActions(row); track action.label) {
<a [routerLink]="action.route">{{ action.label }}</a>
}
</td>
</tr>
}
} @else {
<tr>
<td>{{ row.name }}</td>
<td>{{ row.queue }}</td>
<td><span class="pill" [class]="'pill pill--' + row.state.toLowerCase()">{{ row.state }}</span></td>
<td>{{ row.capacity }}</td>
<td>{{ row.heartbeat }}</td>
<td class="actions">
<a routerLink="/ops/operations/jobs-queues">View</a>
<a routerLink="/ops/operations/jobs-queues">Drain</a>
</td>
<td class="table-empty" colspan="6">No workers match the current filters.</td>
</tr>
}
</tbody>
@@ -274,28 +334,27 @@ interface WorkerRow {
<section class="drawer">
<h2>Context</h2>
@if (tab() === 'jobs') {
<p>Jobs define recurring and ad hoc automation units used by release/security/evidence pipelines.</p>
<p>Job inventory and quota-aware execution control live in JobEngine. Use this overview to route into the right downstream surface.</p>
}
@if (tab() === 'runs') {
<p>
Active issue: <strong>run-004 is in dead-letter</strong> due to upstream feed rate limiting.
Impact: <span class="impact impact--blocking">BLOCKING</span>
Scheduler owns run monitoring. Dead-letter handoff stays explicit so blocking execution incidents do not masquerade as runnable items here.
</p>
}
@if (tab() === 'schedules') {
<p>Schedules control deterministic execution windows and regional workload sequencing.</p>
<p>Schedules own cadence and worker orchestration. Use Scheduler for real edits and worker-fleet coordination.</p>
}
@if (tab() === 'dead-letters') {
<p>
Dead-letter triage is linked to release impact.
<a routerLink="/ops/operations/data-integrity/dlq">Open DLQ & Replays</a>
<a [routerLink]="dataIntegrityDlqPath">Open DLQ recovery</a>
</p>
}
@if (tab() === 'workers') {
<p>Worker capacity and health affect queue latency and decision freshness SLAs.</p>
<p>Worker capacity and health affect queue latency and decision freshness SLAs. Fleet control stays in Scheduler.</p>
}
<div class="drawer__links">
<a routerLink="/ops/operations/data-integrity">Open Data Integrity</a>
<a [routerLink]="OPERATIONS_PATHS.dataIntegrity">Open Data Integrity</a>
<a routerLink="/evidence/audit-log">Open Audit Log</a>
<a routerLink="/releases/runs">Impacted Decisions</a>
</div>
@@ -323,7 +382,7 @@ interface WorkerRow {
margin: 0.2rem 0 0;
font-size: 0.8rem;
color: var(--color-text-secondary);
max-width: 72ch;
max-width: 76ch;
}
.jobs-queues__actions {
@@ -342,6 +401,15 @@ interface WorkerRow {
padding: 0.3rem 0.55rem;
}
.jobs-queues__banner {
border-radius: var(--radius-md);
border: 1px solid var(--color-status-info-border);
background: var(--color-status-info-bg);
color: var(--color-status-info-text);
padding: 0.55rem 0.7rem;
font-size: 0.75rem;
}
.tabs {
display: flex;
gap: 0.35rem;
@@ -386,6 +454,7 @@ interface WorkerRow {
display: flex;
gap: 0.55rem;
flex-wrap: wrap;
align-items: end;
}
.filters label {
@@ -398,7 +467,8 @@ interface WorkerRow {
}
.filters input,
.filters select {
.filters select,
.filters__clear {
border: 1px solid var(--color-border-primary);
border-radius: var(--radius-sm);
background: var(--color-surface-secondary);
@@ -408,6 +478,16 @@ interface WorkerRow {
min-width: 170px;
}
.filters__clear {
min-width: 0;
cursor: pointer;
}
.filters__clear[disabled] {
cursor: default;
opacity: 0.65;
}
.table-wrap {
border: 1px solid var(--color-border-primary);
border-radius: var(--radius-md);
@@ -436,6 +516,13 @@ interface WorkerRow {
color: var(--color-text-secondary);
}
.table-empty {
color: var(--color-text-secondary);
font-style: italic;
text-align: center;
white-space: normal;
}
.pill,
.impact {
border-radius: var(--radius-full);
@@ -489,6 +576,7 @@ interface WorkerRow {
display: flex;
gap: 0.32rem;
align-items: center;
flex-wrap: wrap;
}
.actions a,
@@ -538,16 +626,22 @@ interface WorkerRow {
`],
})
export class PlatformJobsQueuesPageComponent {
readonly OPERATIONS_PATHS = OPERATIONS_PATHS;
readonly dataIntegrityDlqPath = dataIntegrityPath('dlq');
readonly tab = signal<JobsQueuesTab>('jobs');
readonly searchQuery = signal('');
readonly statusFilter = signal('');
readonly facetFilter = signal('');
readonly actionNotice = signal<string | null>(null);
readonly jobs: JobDefinitionRow[] = [
readonly jobs: readonly JobDefinitionRow[] = [
{ id: 'job-def-1', name: 'Container scan', type: 'security', lastRun: '08:03 UTC (2m)', health: 'OK' },
{ id: 'job-def-2', name: 'SBOM generation', type: 'supply', lastRun: '07:55 UTC (1m)', health: 'OK' },
{ id: 'job-def-3', name: 'Compliance export', type: 'evidence', lastRun: '06:00 UTC (5m)', health: 'OK' },
{ id: 'job-def-4', name: 'Vulnerability sync (NVD)', type: 'feeds', lastRun: '05:12 UTC (FAIL)', health: 'DLQ' },
];
readonly runs: JobRunRow[] = [
readonly runs: readonly JobRunRow[] = [
{
id: 'run-001',
job: 'Container scan',
@@ -586,13 +680,13 @@ export class PlatformJobsQueuesPageComponent {
},
];
readonly schedules: ScheduleRow[] = [
{ id: 'sch-1', name: 'Nightly supply scan', cron: '0 2 * * *', nextRun: '02:00 UTC', lastStatus: 'OK' },
{ id: 'sch-2', name: 'Advisory sync', cron: '*/30 * * * *', nextRun: '22:30 UTC', lastStatus: 'WARN' },
{ id: 'sch-3', name: 'Evidence export', cron: '0 6 * * *', nextRun: '06:00 UTC', lastStatus: 'FAIL' },
readonly schedules: readonly ScheduleRow[] = [
{ id: 'sch-1', name: 'Nightly supply scan', cron: '0 2 * * *', nextRun: '02:00 UTC', lastStatus: 'OK', cadence: 'Daily' },
{ id: 'sch-2', name: 'Advisory sync', cron: '*/30 * * * *', nextRun: '22:30 UTC', lastStatus: 'WARN', cadence: 'Hourly' },
{ id: 'sch-3', name: 'Evidence export', cron: '0 6 * * *', nextRun: '06:00 UTC', lastStatus: 'FAIL', cadence: 'Daily' },
];
readonly deadLetters: DeadLetterRow[] = [
readonly deadLetters: readonly DeadLetterRow[] = [
{
id: 'dlq-1',
timestamp: '05:12 UTC',
@@ -622,19 +716,249 @@ export class PlatformJobsQueuesPageComponent {
},
];
readonly workers: WorkerRow[] = [
readonly workers: readonly WorkerRow[] = [
{ id: 'wrk-1', name: 'worker-east-01', queue: 'security', state: 'HEALTHY', capacity: '8/10', heartbeat: '5s ago' },
{ id: 'wrk-2', name: 'worker-east-02', queue: 'feeds', state: 'DEGRADED', capacity: '10/10', heartbeat: '24s ago' },
{ id: 'wrk-3', name: 'worker-eu-01', queue: 'supply', state: 'HEALTHY', capacity: '6/10', heartbeat: '7s ago' },
];
readonly searchPlaceholder = computed(() => {
switch (this.tab()) {
case 'jobs':
return 'Job name, type, or health';
case 'runs':
return 'Run id, job, or correlation id';
case 'schedules':
return 'Schedule name or cron';
case 'dead-letters':
return 'Job, error, or correlation id';
case 'workers':
return 'Worker, queue, or capacity';
}
});
readonly statusLabel = computed(() => {
switch (this.tab()) {
case 'jobs':
return 'Health';
case 'runs':
return 'Status';
case 'schedules':
return 'Last Status';
case 'dead-letters':
return 'Retryable';
case 'workers':
return 'State';
}
});
readonly facetLabel = computed(() => {
switch (this.tab()) {
case 'jobs':
return 'Type';
case 'runs':
return 'Impact';
case 'schedules':
return 'Cadence';
case 'dead-letters':
return 'Impact';
case 'workers':
return 'Queue';
}
});
readonly statusOptions = computed(() => {
switch (this.tab()) {
case 'jobs':
return ['OK', 'WARN', 'DLQ'];
case 'runs':
return ['RUNNING', 'FAILED', 'DEAD-LETTER', 'COMPLETED'];
case 'schedules':
return ['OK', 'WARN', 'FAIL'];
case 'dead-letters':
return ['YES', 'NO'];
case 'workers':
return ['HEALTHY', 'DEGRADED'];
}
});
readonly facetOptions = computed(() => {
switch (this.tab()) {
case 'jobs':
return ['security', 'supply', 'evidence', 'feeds'];
case 'runs':
return ['BLOCKING', 'DEGRADED', 'INFO'];
case 'schedules':
return ['Hourly', 'Daily'];
case 'dead-letters':
return ['BLOCKING', 'DEGRADED'];
case 'workers':
return ['security', 'feeds', 'supply'];
}
});
readonly hasActiveFilters = computed(() =>
Boolean(this.searchQuery().trim() || this.statusFilter() || this.facetFilter()),
);
readonly filteredJobs = computed(() => {
const search = this.normalizedSearch();
const status = this.statusFilter();
const facet = this.facetFilter();
return this.jobs.filter((row) =>
this.matchesSearch(search, [row.id, row.name, row.type, row.health, row.lastRun])
&& this.matchesFilter(status, row.health)
&& this.matchesFilter(facet, row.type),
);
});
readonly filteredRuns = computed(() => {
const search = this.normalizedSearch();
const status = this.statusFilter();
const facet = this.facetFilter();
return this.runs.filter((row) =>
this.matchesSearch(search, [row.id, row.job, row.status, row.impact, row.correlationId, row.startedAt])
&& this.matchesFilter(status, row.status)
&& this.matchesFilter(facet, row.impact),
);
});
readonly filteredSchedules = computed(() => {
const search = this.normalizedSearch();
const status = this.statusFilter();
const facet = this.facetFilter();
return this.schedules.filter((row) =>
this.matchesSearch(search, [row.id, row.name, row.cron, row.nextRun, row.lastStatus, row.cadence])
&& this.matchesFilter(status, row.lastStatus)
&& this.matchesFilter(facet, row.cadence),
);
});
readonly filteredDeadLetters = computed(() => {
const search = this.normalizedSearch();
const status = this.statusFilter();
const facet = this.facetFilter();
return this.deadLetters.filter((row) =>
this.matchesSearch(search, [row.id, row.job, row.error, row.retryable, row.impact, row.correlationId, row.timestamp])
&& this.matchesFilter(status, row.retryable)
&& this.matchesFilter(facet, row.impact),
);
});
readonly filteredWorkers = computed(() => {
const search = this.normalizedSearch();
const status = this.statusFilter();
const facet = this.facetFilter();
return this.workers.filter((row) =>
this.matchesSearch(search, [row.id, row.name, row.queue, row.state, row.capacity, row.heartbeat])
&& this.matchesFilter(status, row.state)
&& this.matchesFilter(facet, row.queue),
);
});
setTab(tab: JobsQueuesTab): void {
if (this.tab() === tab) {
return;
}
this.tab.set(tab);
this.clearFilters();
this.actionNotice.set(null);
}
clearFilters(): void {
this.searchQuery.set('');
this.statusFilter.set('');
this.facetFilter.set('');
}
runsByStatus(status: JobRunRow['status']): number {
return this.runs.filter((row) => row.status === status).length;
}
copyCorrelationId(correlationId: string): void {
if (typeof navigator !== 'undefined' && navigator.clipboard) {
void navigator.clipboard.writeText(correlationId).catch(() => null);
if (typeof navigator === 'undefined' || !navigator.clipboard?.writeText) {
this.actionNotice.set(`Clipboard unavailable. Copy correlation ID ${correlationId} manually.`);
return;
}
void navigator.clipboard.writeText(correlationId)
.then(() => {
this.actionNotice.set(`Copied correlation ID ${correlationId}.`);
})
.catch(() => {
this.actionNotice.set(`Clipboard unavailable. Copy correlation ID ${correlationId} manually.`);
});
}
jobActions(row: JobDefinitionRow): readonly TabAction[] {
return [
{ label: 'Open JobEngine', route: OPERATIONS_PATHS.jobEngine },
{
label: row.health === 'DLQ' ? 'Open Dead-Letter Queue' : 'View Jobs',
route: row.health === 'DLQ' ? deadLetterQueuePath() : OPERATIONS_PATHS.jobEngineJobs,
},
];
}
runActions(row: JobRunRow): readonly TabAction[] {
return [
{ label: 'Open Scheduler Runs', route: OPERATIONS_PATHS.schedulerRuns },
{
label: row.status === 'DEAD-LETTER' ? 'Open Dead-Letter Queue' : 'Open JobEngine Jobs',
route: row.status === 'DEAD-LETTER' ? deadLetterQueuePath() : OPERATIONS_PATHS.jobEngineJobs,
},
];
}
scheduleActions(row: ScheduleRow): readonly TabAction[] {
return [
{ label: 'Manage Schedules', route: OPERATIONS_PATHS.schedulerSchedules },
{
label: row.lastStatus === 'FAIL' ? 'Review Dead-Letter Queue' : 'Open Worker Fleet',
route: row.lastStatus === 'FAIL' ? deadLetterQueuePath() : OPERATIONS_PATHS.schedulerWorkers,
},
];
}
deadLetterActions(row: DeadLetterRow): readonly TabAction[] {
return [
{ label: 'Open Dead-Letter Queue', route: deadLetterQueuePath() },
{
label: row.retryable === 'YES' ? 'Open Replay Recovery' : 'Open JobEngine Jobs',
route: row.retryable === 'YES' ? dataIntegrityPath('dlq') : OPERATIONS_PATHS.jobEngineJobs,
},
];
}
workerActions(row: WorkerRow): readonly TabAction[] {
return [
{ label: 'Open Worker Fleet', route: OPERATIONS_PATHS.schedulerWorkers },
{
label: row.state === 'DEGRADED' ? 'Inspect Scheduler Runs' : 'Open Scheduler Runs',
route: OPERATIONS_PATHS.schedulerRuns,
},
];
}
private normalizedSearch(): string {
return this.searchQuery().trim().toLowerCase();
}
private matchesSearch(search: string, values: readonly string[]): boolean {
if (!search) {
return true;
}
return values.some((value) => value.toLowerCase().includes(search));
}
private matchesFilter(selected: string, actual: string): boolean {
return !selected || selected === actual;
}
}

1
src/Web/StellaOps.Web/tsconfig.spec.features.json
View File

@@ -9,6 +9,7 @@
"src/app/features/deploy-diff/components/deploy-diff-panel/deploy-diff-panel.component.spec.ts",
"src/app/features/deploy-diff/pages/deploy-diff.page.spec.ts",
"src/app/features/deploy-diff/services/deploy-diff.service.spec.ts",
"src/app/features/platform/ops/platform-jobs-queues-page.component.spec.ts",
"src/app/features/policy-simulation/policy-simulation-direct-route-defaults.spec.ts",
"src/app/features/policy-simulation/policy-simulation-defaults.spec.ts",
"src/app/features/policy-simulation/simulation-dashboard.component.spec.ts",