up

ops/devops/orchestrator/alerts.yaml

@@ -28,3 +28,42 @@ groups:
         annotations:
           summary: "Leases stalled"
           description: "No lease renewals while queue has items"
+      - alert: OrchestratorDLQDepthHigh
+        expr: job_dlq_depth > 10
+        for: 10m
+        labels:
+          severity: warning
+          service: orchestrator
+        annotations:
+          summary: "DLQ depth high"
+          description: "Dead-letter queue depth above 10 for 10m"
+      - alert: OrchestratorBackpressure
+        expr: avg_over_time(rate_limiter_backpressure_ratio[5m]) > 0.5
+        for: 5m
+        labels:
+          severity: warning
+          service: orchestrator
+        annotations:
+          summary: "Backpressure elevated"
+          description: "Rate limiter backpressure >50% over 5m"
+      - alert: OrchestratorErrorCluster
+        expr: sum by(jobType) (rate(job_failures_total[5m])) > 3
+        for: 5m
+        labels:
+          severity: critical
+          service: orchestrator
+        annotations:
+          summary: "Error cluster detected"
+          description: "Failure rate >3/min for a job type"
+      - alert: OrchestratorFailureBurnRateHigh
+        expr: |
+          (rate(job_failures_total[5m]) / clamp_min(rate(job_processed_total[5m]), 1)) > 0.02
+          and
+          (rate(job_failures_total[30m]) / clamp_min(rate(job_processed_total[30m]), 1)) > 0.01
+        for: 10m
+        labels:
+          severity: critical
+          service: orchestrator
+        annotations:
+          summary: "Failure burn rate breaching SLO"
+          description: "5m/30m failure burn rate above 2%/1% SLO; investigate upstream jobs and dependencies."