up

ops/devops/airgap/README.md

@@ -12,5 +12,11 @@ Artifacts supporting `DEVOPS-AIRGAP-56-001`:
 - `build_mirror_bundle.py` — Generates mirror bundle manifest + checksums with dual-control approvals; optional cosign signing. Outputs `mirror-bundle-manifest.json`, `checksums.sha256`, and optional signature/cert.
 - `compose-syslog-smtp.yaml` — Local SMTP (MailHog) + syslog-ng stack for sealed environments.
 - `health_syslog_smtp.sh` — Brings up the syslog/SMTP stack via docker compose and performs health checks (MailHog API + syslog logger).
+- `compose-observability.yaml` — Sealed-mode observability stack (Prometheus, Grafana, Tempo, Loki) with offline configs and healthchecks.
+- `health_observability.sh` — Starts the observability stack and probes Prometheus/Grafana/Tempo/Loki readiness.
+- `compose-syslog-smtp.yaml` + `syslog-ng.conf` — Local SMTP + syslog stack for sealed-mode notifications; run via `scripts/devops/run-smtp-syslog.sh` (health check `health_syslog_smtp.sh`).
+- `observability-offline-compose.yml` + `otel-offline.yaml` + `promtail-config.yaml` — Sealed-mode observability stack (Loki, Promtail, OTEL collector with file exporters) to satisfy DEVOPS-AIRGAP-58-002.
+- `compose-syslog-smtp.yaml` — Local SMTP (MailHog) + syslog-ng stack for sealed environments.
+- `health_syslog_smtp.sh` — Brings up the syslog/SMTP stack via docker compose and performs health checks (MailHog API + syslog logger).
 
 See also `ops/devops/sealed-mode-ci/` for the full sealed-mode compose harness and `egress_probe.py`, which this verification script wraps.

ops/devops/airgap/compose-observability.yaml

@@ -0,0 +1,77 @@
+version: "3.9"
+
+services:
+  prometheus:
+    image: prom/prometheus:v2.53.0
+    container_name: prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    volumes:
+      - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    ports:
+      - "9090:9090"
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:9090/-/ready"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
+    restart: unless-stopped
+
+  loki:
+    image: grafana/loki:3.0.0
+    container_name: loki
+    command: ["-config.file=/etc/loki/config.yaml"]
+    volumes:
+      - ./observability/loki-config.yaml:/etc/loki/config.yaml:ro
+      - ./observability/data/loki:/loki
+    ports:
+      - "3100:3100"
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3100/ready"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 15s
+    restart: unless-stopped
+
+  tempo:
+    image: grafana/tempo:2.4.1
+    container_name: tempo
+    command: ["-config.file=/etc/tempo/tempo.yaml"]
+    volumes:
+      - ./observability/tempo-config.yaml:/etc/tempo/tempo.yaml:ro
+      - ./observability/data/tempo:/var/tempo
+    ports:
+      - "3200:3200"
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3200/ready"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 15s
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:10.4.2
+    container_name: grafana
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_SECURITY_ADMIN_USER=admin
+    volumes:
+      - ./observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+    ports:
+      - "3000:3000"
+    depends_on:
+      - prometheus
+      - loki
+      - tempo
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
+    restart: unless-stopped

ops/devops/airgap/compose-syslog-smtp.yaml

@@ -1,31 +1,23 @@
-version: "3.9"
-
+version: '3.8'
 services:
   smtp:
-    image: mailhog/mailhog:v1.0.1
-    container_name: mailhog
-    ports:
-      - "1025:1025"   # SMTP (plain)
-      - "8025:8025"   # Web UI
-    healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://localhost:8025/api/v2/health"]
-      interval: 10s
-      timeout: 3s
-      retries: 5
-      start_period: 5s
+    image: bytemark/smtp
     restart: unless-stopped
-
+    environment:
+      - MAILNAME=sealed.local
+    networks: [sealed]
+    ports:
+      - "2525:25"
   syslog:
     image: balabit/syslog-ng:4.7.1
-    container_name: syslog-ng
-    ports:
-      - "514:514/udp"
-      - "514:514/tcp"
-    command: ["/usr/sbin/syslog-ng", "-F", "-p", "/var/run/syslogd.pid"]
-    healthcheck:
-      test: ["CMD", "syslog-ng-ctl", "stats"]
-      interval: 10s
-      timeout: 3s
-      retries: 5
-      start_period: 5s
     restart: unless-stopped
+    command: ["syslog-ng", "-F", "--no-caps"]
+    networks: [sealed]
+    ports:
+      - "5514:514/udp"
+      - "5515:601/tcp"
+    volumes:
+      - ./syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf:ro
+networks:
+  sealed:
+    driver: bridge

ops/devops/airgap/health_observability.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Health check for compose-observability.yaml (DEVOPS-AIRGAP-58-002)
+
+COMPOSE_FILE="$(cd "$(dirname "$0")" && pwd)/compose-observability.yaml"
+
+echo "Starting observability stack (Prometheus/Grafana/Tempo/Loki)..."
+docker compose -f "$COMPOSE_FILE" up -d
+
+echo "Waiting for containers to report healthy..."
+docker compose -f "$COMPOSE_FILE" wait >/dev/null 2>&1 || true
+
+docker compose -f "$COMPOSE_FILE" ps
+
+echo "Probing Prometheus /-/ready"
+curl -sf http://127.0.0.1:9090/-/ready
+
+echo "Probing Grafana /api/health"
+curl -sf http://127.0.0.1:3000/api/health
+
+echo "Probing Loki /ready"
+curl -sf http://127.0.0.1:3100/ready
+
+echo "Probing Tempo /ready"
+curl -sf http://127.0.0.1:3200/ready
+
+echo "All probes succeeded."

ops/devops/airgap/health_syslog_smtp.sh

@@ -1,23 +1,33 @@
 #!/usr/bin/env bash
 set -euo pipefail
-
 # Health check for compose-syslog-smtp.yaml (DEVOPS-AIRGAP-58-001)
+ROOT=${ROOT:-$(git rev-parse --show-toplevel)}
+COMPOSE_FILE="${COMPOSE_FILE:-$ROOT/ops/devops/airgap/compose-syslog-smtp.yaml}"
+SMTP_PORT=${SMTP_PORT:-2525}
+SYSLOG_TCP=${SYSLOG_TCP:-5515}
+SYSLOG_UDP=${SYSLOG_UDP:-5514}
 
-COMPOSE_FILE="$(cd "$(dirname "$0")" && pwd)/compose-syslog-smtp.yaml"
+export COMPOSE_FILE
+# ensure stack up
+if ! docker compose ps >/dev/null 2>&1; then
+  docker compose up -d
+fi
+sleep 2
 
-echo "Starting syslog+smtp stack..."
-docker compose -f "$COMPOSE_FILE" up -d
+# probe smtp banner
+if ! timeout 5 bash -lc "echo QUIT | nc -w2 127.0.0.1 ${SMTP_PORT}" >/dev/null 2>&1; then
+  echo "smtp service not responding on ${SMTP_PORT}" >&2
+  exit 1
+fi
+# probe syslog tcp
+if ! echo "test" | nc -w2 127.0.0.1 ${SYSLOG_TCP} >/dev/null 2>&1; then
+  echo "syslog tcp not responding on ${SYSLOG_TCP}" >&2
+  exit 1
+fi
+# probe syslog udp
+if ! echo "test" | nc -w2 -u 127.0.0.1 ${SYSLOG_UDP} >/dev/null 2>&1; then
+  echo "syslog udp not responding on ${SYSLOG_UDP}" >&2
+  exit 1
+fi
 
-echo "Waiting for health checks..."
-docker compose -f "$COMPOSE_FILE" wait >/dev/null 2>&1 || true
-
-echo "Current health status:"
-docker compose -f "$COMPOSE_FILE" ps
-
-echo "Sending test syslog message (UDP)..."
-logger -n 127.0.0.1 -P 514 -d "test syslog message $(date -u +%s)"
-
-echo "SMTP health endpoint:"
-curl -sf http://127.0.0.1:8025/api/v2/health || exit 1
-
-echo "Done."
+echo "smtp/syslog stack healthy"

ops/devops/airgap/observability-offline-compose.yml

@@ -0,0 +1,32 @@
+version: '3.8'
+services:
+  loki:
+    image: grafana/loki:3.0.1
+    command: ["-config.file=/etc/loki/local-config.yaml"]
+    volumes:
+      - loki-data:/loki
+    networks: [sealed]
+  promtail:
+    image: grafana/promtail:3.0.1
+    command: ["-config.file=/etc/promtail/config.yml"]
+    volumes:
+      - promtail-data:/var/log
+      - ./promtail-config.yaml:/etc/promtail/config.yml:ro
+    networks: [sealed]
+  otel:
+    image: otel/opentelemetry-collector-contrib:0.97.0
+    command: ["--config=/etc/otel/otel-offline.yaml"]
+    volumes:
+      - ./otel-offline.yaml:/etc/otel/otel-offline.yaml:ro
+      - otel-data:/var/otel
+    ports:
+      - "4317:4317"
+      - "4318:4318"
+    networks: [sealed]
+networks:
+  sealed:
+    driver: bridge
+volumes:
+  loki-data:
+  promtail-data:
+  otel-data:

ops/devops/airgap/observability/grafana/provisioning/datasources/datasources.yaml

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200

ops/devops/airgap/observability/loki-config.yaml

@@ -0,0 +1,35 @@
+server:
+  http_listen_port: 3100
+  log_level: warn
+
+common:
+  ring:
+    instance_addr: loki
+    kvstore:
+      store: inmemory
+  replication_factor: 1
+
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 168h
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  filesystem:
+    directory: /loki/chunks
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/cache
+    shared_store: filesystem
+
+limits_config:
+  retention_period: 168h

ops/devops/airgap/observability/prometheus.yml

@@ -0,0 +1,14 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['prometheus:9090']
+  - job_name: loki
+    static_configs:
+      - targets: ['loki:3100']
+  - job_name: tempo
+    static_configs:
+      - targets: ['tempo:3200']

ops/devops/airgap/observability/tempo-config.yaml

@@ -0,0 +1,26 @@
+server:
+  http_listen_port: 3200
+  log_level: warn
+
+distributor:
+  receivers:
+    jaeger:
+      protocols:
+        thrift_http:
+    otlp:
+      protocols:
+        http:
+        grpc:
+    zipkin:
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /var/tempo/wal
+    local:
+      path: /var/tempo/traces
+
+compactor:
+  compaction:
+    block_retention: 168h

ops/devops/airgap/otel-offline.yaml

@@ -0,0 +1,40 @@
+receivers:
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'self'
+          static_configs:
+            - targets: ['localhost:8888']
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 512
+exporters:
+  file/metrics:
+    path: /var/otel/metrics.prom
+  file/traces:
+    path: /var/otel/traces.ndjson
+  loki/offline:
+    endpoint: http://loki:3100/loki/api/v1/push
+    labels:
+      job: sealed-observability
+    tenant_id: "sealed"
+service:
+  telemetry:
+    logs:
+      level: info
+  pipelines:
+    metrics:
+      receivers: [prometheus]
+      processors: [batch]
+      exporters: [file/metrics]
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [file/traces]

ops/devops/airgap/promtail-config.yaml

@@ -0,0 +1,14 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+positions:
+  filename: /tmp/positions.yaml
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+scrape_configs:
+  - job_name: promtail
+    static_configs:
+      - targets: [localhost]
+        labels:
+          job: promtail
+          __path__: /var/log/*.log

ops/devops/airgap/syslog-ng.conf

@@ -0,0 +1,19 @@
+@version: 4.7
+@include "scl.conf"
+
+options {
+    time-reopen(10);
+    log-msg-size(8192);
+    ts-format(iso);
+};
+
+source s_net {
+    tcp(port(601));
+    udp(port(514));
+};
+
+destination d_file {
+    file("/var/log/syslog-ng/sealed.log" create-dirs(yes) perm(0644));
+};
+
+log { source(s_net); destination(d_file); };