git.stella-ops.org/src/Web/StellaOps.Web/tests/e2e/execution-operations-cutover.spec.ts

import { expect, test, type Page, type Route } from '@playwright/test';

import type { StubAuthSession } from '../../src/app/testing/auth-fixtures';

const operatorSession: StubAuthSession = {
  subjectId: 'ops-e2e-user',
  tenant: 'tenant-default',
  scopes: [
    'admin',
    'ui.read',
    'ui.admin',
    'orch:read',
    'orch:operate',
    'orch:quota',
    'orch:backfill',
    'health:read',
    'scheduler:read',
    'scheduler:trigger',
    'policy:read',
    'release:read',
  ],
};

const mockConfig = {
  authority: {
    issuer: '/authority',
    clientId: 'stella-ops-ui',
    authorizeEndpoint: '/authority/connect/authorize',
    tokenEndpoint: '/authority/connect/token',
    logoutEndpoint: '/authority/connect/logout',
    redirectUri: 'https://127.0.0.1:4400/auth/callback',
    postLogoutRedirectUri: 'https://127.0.0.1:4400/',
    scope: 'openid profile email ui.read',
    audience: '/gateway',
    dpopAlgorithms: ['ES256'],
    refreshLeewaySeconds: 60,
  },
  apiBaseUrls: {
    authority: '/authority',
    scanner: '/scanner',
    policy: '/policy',
    concelier: '/concelier',
    attestor: '/attestor',
    gateway: '/gateway',
  },
  quickstartMode: true,
  setup: 'complete',
};

const jobList = {
  jobs: [
    {
      tenantId: 'tenant-default',
      projectId: 'proj-a',
      jobId: 'job-001',
      runId: 'run-001',
      jobType: 'scan',
      status: 'failed',
      priority: 10,
      attempt: 2,
      maxAttempts: 3,
      correlationId: 'corr-001',
      workerId: 'worker-1',
      taskRunnerId: 'runner-a',
      createdAt: '2026-03-08T08:00:00Z',
      scheduledAt: '2026-03-08T08:00:00Z',
      leasedAt: '2026-03-08T08:01:00Z',
      completedAt: '2026-03-08T08:05:00Z',
      notBefore: '2026-03-08T08:00:00Z',
      reason: 'Dead-letter candidate',
      replayOf: null,
      createdBy: 'operator',
    },
  ],
  nextCursor: null,
};

const jobDetail = {
  ...jobList.jobs[0],
  payloadDigest: 'sha256:payload-001',
  payload: JSON.stringify({ artifact: 'sha256:artifact-001', action: 'scan' }),
  idempotencyKey: 'idem-001',
  leaseId: 'lease-001',
  leaseUntil: '2026-03-08T08:06:00Z',
};

const quotaList = {
  items: [
    {
      quotaId: 'quota-001',
      tenantId: 'tenant-default',
      jobType: 'scan',
      maxActive: 4,
      maxPerHour: 100,
      burstCapacity: 100,
      refillRate: 5,
      currentTokens: 40,
      currentActive: 1,
      currentHourCount: 10,
      paused: false,
      pauseReason: null,
      quotaTicket: null,
      createdAt: '2026-03-01T00:00:00Z',
      updatedAt: '2026-03-08T07:30:00Z',
      updatedBy: 'ops@example.com',
    },
  ],
  count: 1,
  continuationToken: null,
};

const quotaSummary = {
  totalQuotas: 1,
  pausedQuotas: 0,
  averageTokenUtilization: 0.6,
  averageConcurrencyUtilization: 0.25,
};

const jobSummary = {
  totalJobs: 12,
  pendingJobs: 2,
  scheduledJobs: 3,
  leasedJobs: 1,
  succeededJobs: 5,
  failedJobs: 1,
  canceledJobs: 1,
  timedOutJobs: 0,
};

const deadLetterEntries = [
  {
    id: 'dlq-001',
    jobId: 'job-001',
    jobType: 'scan',
    tenantId: 'tenant-default',
    tenantName: 'Default Tenant',
    state: 'failed',
    errorCode: 'DLQ_TIMEOUT',
    errorMessage: 'Scanner timed out',
    retryCount: 2,
    maxRetries: 3,
    age: 7200,
    createdAt: '2026-03-08T06:00:00Z',
  },
];

const schedulerRuns = [
  {
    id: 'run-001',
    scheduleId: 'sch-001',
    scheduleName: 'Nightly Sync',
    status: 'running',
    triggeredAt: '2026-03-08T07:00:00Z',
    startedAt: '2026-03-08T07:01:00Z',
    completedAt: null,
    durationMs: null,
    triggeredBy: 'schedule',
    progress: 48,
    itemsProcessed: 48,
    itemsTotal: 100,
    retryCount: 0,
    output: { metrics: { scanned: 48, failed: 1 } },
  },
];

async function fulfillJson(route: Route, body: unknown, status = 200): Promise<void> {
  await route.fulfill({
    status,
    contentType: 'application/json',
    body: JSON.stringify(body),
  });
}

async function setupHarness(page: Page): Promise<void> {
  await page.addInitScript((session) => {
    (window as { __stellaopsTestSession?: unknown }).__stellaopsTestSession = session;
  }, operatorSession);

  await page.route('**/platform/envsettings.json', (route) => fulfillJson(route, mockConfig));
  await page.route('**/platform/i18n/*.json', (route) => fulfillJson(route, {}));
  await page.route('**/config.json', (route) => fulfillJson(route, mockConfig));
  await page.route('**/.well-known/openid-configuration', (route) =>
    fulfillJson(route, {
      issuer: 'https://127.0.0.1:4400/authority',
      authorization_endpoint: 'https://127.0.0.1:4400/authority/connect/authorize',
      token_endpoint: 'https://127.0.0.1:4400/authority/connect/token',
      jwks_uri: 'https://127.0.0.1:4400/authority/.well-known/jwks.json',
      response_types_supported: ['code'],
      subject_types_supported: ['public'],
      id_token_signing_alg_values_supported: ['RS256'],
    }),
  );
  await page.route('**/authority/.well-known/jwks.json', (route) => fulfillJson(route, { keys: [] }));
  await page.route('**/console/branding**', (route) =>
    fulfillJson(route, {
      tenantId: operatorSession.tenant,
      appName: 'Stella Ops',
      logoUrl: null,
      cssVariables: {},
    }),
  );
  await page.route('**/console/profile**', (route) =>
    fulfillJson(route, {
      subjectId: operatorSession.subjectId,
      username: 'ops-e2e',
      displayName: 'Ops E2E',
      tenant: operatorSession.tenant,
      roles: ['ops-operator'],
      scopes: operatorSession.scopes,
    }),
  );
  await page.route('**/console/token/introspect**', (route) =>
    fulfillJson(route, {
      active: true,
      tenant: operatorSession.tenant,
      subject: operatorSession.subjectId,
      scopes: operatorSession.scopes,
    }),
  );
  await page.route('**/authority/console/tenants**', (route) =>
    fulfillJson(route, {
      tenants: [
        {
          tenantId: operatorSession.tenant,
          displayName: 'Default Tenant',
          isDefault: true,
          isActive: true,
        },
      ],
    }),
  );
  // Register the generic API fallback before exact mocks so the exact handlers win.
  await page.route('**/api/**', (route) => fulfillJson(route, {}));
  await page.route('**/api/v2/context/regions**', (route) =>
    fulfillJson(route, [{ regionId: 'eu-west', displayName: 'EU West', sortOrder: 1, enabled: true }]),
  );
  await page.route('**/api/v2/context/environments**', (route) =>
    fulfillJson(route, [
      {
        environmentId: 'prod',
        regionId: 'eu-west',
        environmentType: 'prod',
        displayName: 'Prod',
        sortOrder: 1,
        enabled: true,
      },
    ]),
  );
  await page.route('**/api/v2/context/preferences**', (route) =>
    fulfillJson(route, {
      tenantId: operatorSession.tenant,
      actorId: operatorSession.subjectId,
      regions: ['eu-west'],
      environments: ['prod'],
      timeWindow: '24h',
      stage: 'all',
      updatedAt: '2026-03-08T07:00:00Z',
      updatedBy: operatorSession.subjectId,
    }),
  );
  await page.route('**/doctor/api/v1/doctor/trends**', (route) => fulfillJson(route, []));
  await page.route('**/api/v1/approvals**', (route) => fulfillJson(route, []));
  await page.route('**/api/v1/telemetry/ttfs', (route) => fulfillJson(route, { accepted: true }, 202));

  await page.route(/\/gateway\/jobengine\/jobs\/summary(?:\?.*)?$/, (route) => fulfillJson(route, jobSummary));
  await page.route(/\/gateway\/jobengine\/jobs(?:\?.*)?$/, (route) => fulfillJson(route, jobList));
  await page.route(/\/gateway\/jobengine\/jobs\/job-001\/detail(?:\?.*)?$/, (route) => fulfillJson(route, jobDetail));
  await page.route(/\/gateway\/jobengine\/dag\/job\/job-001\/parents(?:\?.*)?$/, (route) =>
    fulfillJson(route, { edges: [] }),
  );
  await page.route(/\/gateway\/jobengine\/dag\/job\/job-001\/children(?:\?.*)?$/, (route) =>
    fulfillJson(route, {
      edges: [
        {
          edgeId: 'edge-001',
          runId: 'run-001',
          parentJobId: 'job-001',
          childJobId: 'job-002',
          edgeType: 'dependency',
          createdAt: '2026-03-08T08:02:00Z',
        },
      ],
    }),
  );
  await page.route(/\/gateway\/jobengine\/quotas\/summary(?:\?.*)?$/, (route) => fulfillJson(route, quotaSummary));
  await page.route(/\/gateway\/jobengine\/quotas(?:\?.*)?$/, (route) => fulfillJson(route, quotaList));
  await page.route(/\/gateway\/jobengine\/quotas\/quota-001\/pause(?:\?.*)?$/, (route) =>
    fulfillJson(route, { ...quotaList.items[0], paused: true, pauseReason: 'Paused from execution operations UI' }),
  );
  await page.route(/\/gateway\/jobengine\/deadletter\/stats(?:\?.*)?$/, (route) =>
    fulfillJson(route, {
      totalEntries: 1,
      pendingEntries: 0,
      replayingEntries: 0,
      replayedEntries: 0,
      resolvedEntries: 0,
      exhaustedEntries: 1,
      expiredEntries: 0,
      retryableEntries: 1,
      topErrorCodes: { DLQ_TIMEOUT: 1 },
    }),
  );

  await page.route(/\/api\/v1\/jobengine\/deadletter\/stats(?:\?.*)?$/, (route) =>
    fulfillJson(route, {
      totalEntries: 1,
      pendingEntries: 0,
      replayingEntries: 0,
      replayedEntries: 0,
      resolvedEntries: 0,
      exhaustedEntries: 1,
      expiredEntries: 0,
      retryableEntries: 1,
      topErrorCodes: { DLQ_TIMEOUT: 1 },
    }),
  );
  await page.route(/\/api\/v1\/jobengine\/deadletter\/summary(?:\?.*)?$/, (route) =>
    fulfillJson(route, { summaries: [{ errorCode: 'DLQ_TIMEOUT', entryCount: 1 }] }),
  );
  await page.route(/\/api\/v1\/jobengine\/deadletter\/resolve\/batch(?:\?.*)?$/, (route) =>
    fulfillJson(route, { resolvedCount: 1 }),
  );
  await page.route(/\/api\/v1\/jobengine\/deadletter\/dlq-001\/replay(?:\?.*)?$/, (route) =>
    fulfillJson(route, { success: true, newJobId: 'job-replay-001' }),
  );
  await page.route(/\/api\/v1\/jobengine\/deadletter(?:\?.*)?$/, (route) =>
    fulfillJson(route, { items: deadLetterEntries, total: deadLetterEntries.length }),
  );

  await page.route('**/scheduler/api/v1/scheduler/runs**', (route) =>
    fulfillJson(route, schedulerRuns),
  );
}

test.beforeEach(async ({ page }) => {
  await setupHarness(page);
});

test('execution operations cutover keeps canonical jobengine, dead-letter, scheduler, and scanner flows usable', async ({
  page,
}) => {
  await page.goto('/ops/operations/jobengine/jobs', { waitUntil: 'networkidle' });

  await expect(page.getByRole('heading', { name: 'JobEngine Jobs' })).toBeVisible();
  await page.locator('.job-card').first().getByRole('button', { name: 'Expand', exact: true }).click();
  await page.getByRole('link', { name: 'View Details' }).click();
  await expect(page).toHaveURL(/\/ops\/operations\/jobengine\/jobs\/job-001$/);
  await expect(page.getByText('Execution Metadata')).toBeVisible();

  await page.goto('/ops/operations/dead-letter/queue', { waitUntil: 'networkidle' });
  await expect(page.getByRole('heading', { name: 'Dead-Letter Queue' })).toBeVisible();
  await page.locator('tbody input[type="checkbox"]').first().check();
  await page.getByRole('button', { name: 'Replay Selected' }).click();
  await expect(page.getByText('Queued replay for 1 selected entry.')).toBeVisible();

  await page.goto('/ops/operations/scheduler/runs', { waitUntil: 'networkidle' });
  await expect(page.getByRole('heading', { name: 'Scheduler Runs' })).toBeVisible();
  await page.getByText('Nightly Sync').click();
  await page.getByRole('link', { name: 'Live Stream' }).click();
  await expect(page).toHaveURL(/\/ops\/operations\/scheduler\/runs\/run-001\/stream$/);
  await expect(page.getByRole('heading', { name: 'Scheduler Run Stream' })).toBeVisible();

  await page.goto('/ops/scanner-ops/offline-kits', { waitUntil: 'networkidle' });
  await expect(page.getByRole('heading', { name: 'Scanner Operations' })).toBeVisible();
  await page.getByRole('button', { name: 'Verify All' }).click();
  await expect(page.getByText('Verified 2 kits locally.')).toBeVisible();
});