Add tenant isolation smoke test for telemetry stack

This commit introduces a new script `tenant_isolation_smoke.py` that performs smoke tests to validate tenant isolation in the telemetry storage stack (Tempo + Loki) with mutual TLS enabled. The script checks that traces and logs pushed with specific tenant headers are only accessible to the corresponding tenants, ensuring proper enforcement of multi-tenancy. The tests include pushing a trace and a log entry, followed by assertions to verify access restrictions based on tenant IDs.

deploy/compose/docker-compose.telemetry.yaml

@@ -8,11 +8,19 @@ services:
       - "--config=/etc/otel-collector/config.yaml"
     environment:
       STELLAOPS_OTEL_TLS_CERT: /etc/otel-collector/tls/collector.crt
-      STELLAOPS_OTEL_TLS_KEY: /etc/otel-collector/tls/collector.key
-      STELLAOPS_OTEL_TLS_CA: /etc/otel-collector/tls/ca.crt
-      STELLAOPS_OTEL_PROMETHEUS_ENDPOINT: 0.0.0.0:9464
-      STELLAOPS_OTEL_REQUIRE_CLIENT_CERT: "true"
-      STELLAOPS_TENANT_ID: dev
+      STELLAOPS_OTEL_TLS_KEY: /etc/otel-collector/tls/collector.key
+      STELLAOPS_OTEL_TLS_CA: /etc/otel-collector/tls/ca.crt
+      STELLAOPS_OTEL_PROMETHEUS_ENDPOINT: 0.0.0.0:9464
+      STELLAOPS_OTEL_REQUIRE_CLIENT_CERT: "true"
+      STELLAOPS_TENANT_ID: dev
+      STELLAOPS_TEMPO_ENDPOINT: https://stellaops-tempo:3200
+      STELLAOPS_TEMPO_TLS_CERT_FILE: /etc/otel-collector/tls/client.crt
+      STELLAOPS_TEMPO_TLS_KEY_FILE: /etc/otel-collector/tls/client.key
+      STELLAOPS_TEMPO_TLS_CA_FILE: /etc/otel-collector/tls/ca.crt
+      STELLAOPS_LOKI_ENDPOINT: https://stellaops-loki:3100/loki/api/v1/push
+      STELLAOPS_LOKI_TLS_CERT_FILE: /etc/otel-collector/tls/client.crt
+      STELLAOPS_LOKI_TLS_KEY_FILE: /etc/otel-collector/tls/client.key
+      STELLAOPS_LOKI_TLS_CA_FILE: /etc/otel-collector/tls/ca.crt
     volumes:
       - ../telemetry/otel-collector-config.yaml:/etc/otel-collector/config.yaml:ro
       - ../telemetry/certs:/etc/otel-collector/tls:ro

deploy/telemetry/otel-collector-config.yaml

@@ -26,20 +26,45 @@ processors:
     send_batch_size: 1024
     timeout: 5s
 
-exporters:
-  logging:
-    verbosity: normal
-  prometheus:
-    endpoint: ${STELLAOPS_OTEL_PROMETHEUS_ENDPOINT:0.0.0.0:9464}
-    enable_open_metrics: true
-    metric_expiration: 5m
-    tls:
-      cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set}
-      key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set}
-      client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set}
-# Additional OTLP exporters can be configured by extending this section at runtime.
-# For example, set STELLAOPS_OTEL_UPSTREAM_ENDPOINT and mount certificates, then
-# add the exporter via a sidecar overlay.
+exporters:
+  logging:
+    verbosity: normal
+  prometheus:
+    endpoint: ${STELLAOPS_OTEL_PROMETHEUS_ENDPOINT:0.0.0.0:9464}
+    enable_open_metrics: true
+    metric_expiration: 5m
+    tls:
+      cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set}
+      key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set}
+      client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set}
+  otlphttp/tempo:
+    endpoint: ${STELLAOPS_TEMPO_ENDPOINT:https://stellaops-tempo:3200}
+    compression: gzip
+    tls:
+      ca_file: ${STELLAOPS_TEMPO_TLS_CA_FILE:/etc/otel-collector/tls/ca.crt}
+      cert_file: ${STELLAOPS_TEMPO_TLS_CERT_FILE:/etc/otel-collector/tls/client.crt}
+      key_file: ${STELLAOPS_TEMPO_TLS_KEY_FILE:/etc/otel-collector/tls/client.key}
+      insecure_skip_verify: false
+    headers:
+      "X-Scope-OrgID": ${STELLAOPS_TENANT_ID:unknown}
+  loki/tenant:
+    endpoint: ${STELLAOPS_LOKI_ENDPOINT:https://stellaops-loki:3100/loki/api/v1/push}
+    tenant_id: ${STELLAOPS_TENANT_ID:unknown}
+    tls:
+      ca_file: ${STELLAOPS_LOKI_TLS_CA_FILE:/etc/otel-collector/tls/ca.crt}
+      cert_file: ${STELLAOPS_LOKI_TLS_CERT_FILE:/etc/otel-collector/tls/client.crt}
+      key_file: ${STELLAOPS_LOKI_TLS_KEY_FILE:/etc/otel-collector/tls/client.key}
+      insecure_skip_verify: false
+    default_labels_enabled:
+      exporter: false
+      job: false
+      instance: false
+    format: json
+    drain_interval: 5s
+    queue:
+      enabled: true
+      queue_size: 1024
+      retry_on_failure: true
 
 extensions:
   health_check:
@@ -53,15 +78,15 @@ service:
       level: ${STELLAOPS_OTEL_LOG_LEVEL:info}
   extensions: [health_check, pprof]
   pipelines:
-    traces:
-      receivers: [otlp]
-      processors: [attributes/tenant-tag, batch]
-      exporters: [logging]
+    traces:
+      receivers: [otlp]
+      processors: [attributes/tenant-tag, batch]
+      exporters: [logging, otlphttp/tempo]
     metrics:
       receivers: [otlp]
       processors: [attributes/tenant-tag, batch]
       exporters: [logging, prometheus]
-    logs:
-      receivers: [otlp]
-      processors: [attributes/tenant-tag, batch]
-      exporters: [logging]
+    logs:
+      receivers: [otlp]
+      processors: [attributes/tenant-tag, batch]
+      exporters: [logging, loki/tenant]