git.stella-ops.org/ops/devops/findings-ledger/offline-kit/alerts/findings-ledger-alerts.yaml

# Findings Ledger Prometheus Alert Rules
# Apply to Prometheus: cp findings-ledger-alerts.yaml /etc/prometheus/rules.d/

groups:
  - name: findings-ledger
    rules:
      # Service availability
      - alert: FindingsLedgerDown
        expr: up{job="findings-ledger"} == 0
        for: 2m
        labels:
          severity: critical
          service: findings-ledger
        annotations:
          summary: "Findings Ledger service is down"
          description: "Findings Ledger service has been unreachable for more than 2 minutes."

      # Write latency
      - alert: FindingsLedgerHighWriteLatency
        expr: histogram_quantile(0.95, sum(rate(ledger_write_latency_seconds_bucket{job="findings-ledger"}[5m])) by (le)) > 1
        for: 5m
        labels:
          severity: warning
          service: findings-ledger
        annotations:
          summary: "Findings Ledger write latency is high"
          description: "95th percentile write latency exceeds 1 second for 5 minutes. Current: {{ $value | humanizeDuration }}"

      - alert: FindingsLedgerCriticalWriteLatency
        expr: histogram_quantile(0.95, sum(rate(ledger_write_latency_seconds_bucket{job="findings-ledger"}[5m])) by (le)) > 5
        for: 2m
        labels:
          severity: critical
          service: findings-ledger
        annotations:
          summary: "Findings Ledger write latency is critically high"
          description: "95th percentile write latency exceeds 5 seconds. Current: {{ $value | humanizeDuration }}"

      # Projection lag
      - alert: FindingsLedgerProjectionLag
        expr: ledger_projection_lag_seconds{job="findings-ledger"} > 30
        for: 5m
        labels:
          severity: warning
          service: findings-ledger
        annotations:
          summary: "Findings Ledger projection lag is high"
          description: "Projection lag exceeds 30 seconds for 5 minutes. Current: {{ $value | humanizeDuration }}"

      - alert: FindingsLedgerCriticalProjectionLag
        expr: ledger_projection_lag_seconds{job="findings-ledger"} > 300
        for: 2m
        labels:
          severity: critical
          service: findings-ledger
        annotations:
          summary: "Findings Ledger projection lag is critically high"
          description: "Projection lag exceeds 5 minutes. Current: {{ $value | humanizeDuration }}"

      # Merkle anchoring
      - alert: FindingsLedgerMerkleAnchorStale
        expr: time() - ledger_merkle_last_anchor_timestamp_seconds{job="findings-ledger"} > 600
        for: 5m
        labels:
          severity: warning
          service: findings-ledger
        annotations:
          summary: "Findings Ledger Merkle anchor is stale"
          description: "No Merkle anchor created in the last 10 minutes. Last anchor: {{ $value | humanizeTimestamp }}"

      - alert: FindingsLedgerMerkleAnchorFailed
        expr: increase(ledger_merkle_anchor_failures_total{job="findings-ledger"}[15m]) > 0
        for: 0m
        labels:
          severity: warning
          service: findings-ledger
        annotations:
          summary: "Findings Ledger Merkle anchoring failed"
          description: "Merkle anchor operation failed. Check logs for details."

      # Database connectivity
      - alert: FindingsLedgerDatabaseErrors
        expr: increase(ledger_database_errors_total{job="findings-ledger"}[5m]) > 5
        for: 2m
        labels:
          severity: warning
          service: findings-ledger
        annotations:
          summary: "Findings Ledger database errors detected"
          description: "More than 5 database errors in the last 5 minutes."

      # Attachment storage
      - alert: FindingsLedgerAttachmentStorageErrors
        expr: increase(ledger_attachment_storage_errors_total{job="findings-ledger"}[15m]) > 0
        for: 0m
        labels:
          severity: warning
          service: findings-ledger
        annotations:
          summary: "Findings Ledger attachment storage errors"
          description: "Attachment storage operation failed. Check encryption keys and storage connectivity."

      # Air-gap staleness (for offline environments)
      - alert: FindingsLedgerAdvisoryStaleness
        expr: ledger_airgap_advisory_staleness_seconds{job="findings-ledger"} > 604800
        for: 1h
        labels:
          severity: warning
          service: findings-ledger
        annotations:
          summary: "Advisory data is stale in air-gapped environment"
          description: "Advisory data is older than 7 days. Import fresh data from Mirror."

      - alert: FindingsLedgerVexStaleness
        expr: ledger_airgap_vex_staleness_seconds{job="findings-ledger"} > 604800
        for: 1h
        labels:
          severity: warning
          service: findings-ledger
        annotations:
          summary: "VEX data is stale in air-gapped environment"
          description: "VEX data is older than 7 days. Import fresh data from Mirror."