31 lines
973 B
YAML
31 lines
973 B
YAML
groups:
|
|
- name: orchestrator-core
|
|
rules:
|
|
- alert: OrchestratorQueueDepthHigh
|
|
expr: job_queue_depth > 500
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
service: orchestrator
|
|
annotations:
|
|
summary: "Queue depth high"
|
|
description: "job_queue_depth exceeded 500 for 10m"
|
|
- alert: OrchestratorFailuresHigh
|
|
expr: rate(job_failures_total[5m]) > 5
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: orchestrator
|
|
annotations:
|
|
summary: "Job failures elevated"
|
|
description: "Failure rate above 5/min in last 5m"
|
|
- alert: OrchestratorLeaseStall
|
|
expr: rate(lease_extensions_total[5m]) == 0 and job_queue_depth > 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
service: orchestrator
|
|
annotations:
|
|
summary: "Leases stalled"
|
|
description: "No lease renewals while queue has items"
|