stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

devops folders consolidate

Browse Source
This commit is contained in:
master
2026-01-25 23:27:41 +02:00
parent 6e687b523a
commit a743bb9a1d
613 changed files with 8611 additions and 41846 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
devops/telemetry/.gitignore vendored
View File

@@ -1 +0,0 @@
certs/

35
devops/telemetry/README.md
View File

@@ -1,35 +0,0 @@
# Telemetry Collector Assets
These assets provision the default OpenTelemetry Collector instance required by
`DEVOPS-OBS-50-001`. The collector acts as the secured ingest point for traces,
metrics, and logs emitted by Stella Ops services.
## Contents
| File | Purpose |
| ---- | ------- |
| `otel-collector-config.yaml` | Baseline collector configuration (mutual TLS, OTLP receivers, Prometheus exporter). |
| `storage/prometheus.yaml` | Prometheus scrape configuration tuned for the collector and service tenants. |
| `storage/tempo.yaml` | Tempo configuration with multitenancy, WAL, and compaction settings. |
| `storage/loki.yaml` | Loki configuration enabling multitenant log ingestion with retention policies. |
| `storage/tenants/*.yaml` | Per-tenant overrides for Tempo and Loki rate/retention controls. |
## Development workflow
1. Generate development certificates (collector + client) using
`ops/devops/telemetry/generate_dev_tls.sh`.
2. Launch the collector via `docker compose -f docker-compose.telemetry.yaml up`.
3. Launch the storage backends (Prometheus, Tempo, Loki) via
`docker compose -f docker-compose.telemetry-storage.yaml up`.
4. Run the smoke test: `python ops/devops/telemetry/smoke_otel_collector.py`.
5. Explore the storage configuration (`storage/README.md`) to tune retention/limits.
The smoke test sends OTLP traffic over TLS and asserts the collector accepted
traces, metrics, and logs by scraping the Prometheus metrics endpoint.
## Kubernetes
The Helm chart consumes the same configuration (see `values.yaml`). Provide TLS
material via a secret referenced by `telemetry.collector.tls.secretName`,
containing `ca.crt`, `tls.crt`, and `tls.key`. Client certificates are required
for ingestion and should be issued by the same CA.

36
devops/telemetry/alerts/alerts-slo.yaml Normal file
View File

@@ -0,0 +1,36 @@
groups:
- name: slo-burn
rules:
- alert: SLOBurnRateFast
expr: |
(rate(service_request_errors_total[5m]) / rate(service_requests_total[5m])) >
4 * (1 - 0.99)
for: 5m
labels:
severity: critical
team: devops
annotations:
summary: "Fast burn: 99% SLO breached"
description: "Error budget burn (5m) exceeds fast threshold."
- alert: SLOBurnRateSlow
expr: |
(rate(service_request_errors_total[1h]) / rate(service_requests_total[1h])) >
1 * (1 - 0.99)
for: 1h
labels:
severity: warning
team: devops
annotations:
summary: "Slow burn: 99% SLO at risk"
description: "Error budget burn (1h) exceeds slow threshold."
- name: slo-webhook
rules:
- alert: SLOWebhookFailures
expr: rate(slo_webhook_failures_total[5m]) > 0
for: 10m
labels:
severity: warning
team: devops
annotations:
summary: "SLO webhook failures"
description: "Webhook emitter has failures in last 5m."

43
devops/telemetry/alerts/attestation-alerts.yaml Normal file
View File

@@ -0,0 +1,43 @@
groups:
- name: attestor-latency
rules:
- alert: AttestorSignLatencyP95High
expr: histogram_quantile(0.95, sum(rate(attestor_sign_duration_seconds_bucket[5m])) by (le)) > 2
for: 5m
labels:
severity: warning
team: devops
annotations:
summary: "Attestor signing latency p95 high"
description: "Signing p95 is {{ $value }}s over the last 5m (threshold 2s)."
- alert: AttestorVerifyLatencyP95High
expr: histogram_quantile(0.95, sum(rate(attestor_verify_duration_seconds_bucket[5m])) by (le)) > 2
for: 5m
labels:
severity: warning
team: devops
annotations:
summary: "Attestor verification latency p95 high"
description: "Verification p95 is {{ $value }}s over the last 5m (threshold 2s)."
- name: attestor-errors
rules:
- alert: AttestorVerifyFailureRate
expr: rate(attestor_verify_failures_total[5m]) / rate(attestor_verify_requests_total[5m]) > 0.02
for: 5m
labels:
severity: critical
team: devops
annotations:
summary: "Attestor verification failure rate above 2%"
description: "Verification failure rate is {{ $value | humanizePercentage }} over last 5m."
- name: attestor-keys
rules:
- alert: AttestorKeyRotationStale
expr: (time() - attestor_key_last_rotated_seconds) > 60*60*24*30
for: 10m
labels:
severity: warning
team: devops
annotations:
summary: "Attestor signing key rotation overdue"
description: "Signing key has not rotated in >30d ({{ $value }} seconds)."

52
devops/telemetry/alerts/policy-alerts.yaml Normal file
View File

@@ -0,0 +1,52 @@
groups:
- name: policy-pipeline
rules:
- alert: PolicyCompileLatencyP99High
expr: histogram_quantile(0.99, sum(rate(policy_compile_duration_seconds_bucket[5m])) by (le)) > 5
for: 10m
labels:
severity: warning
service: policy
annotations:
summary: "Policy compile latency elevated (p99)"
description: "p99 compile duration has been >5s for 10m"
- alert: PolicySimulationQueueBacklog
expr: sum(policy_simulation_queue_depth) > 100
for: 10m
labels:
severity: warning
service: policy
annotations:
summary: "Policy simulation backlog"
description: "Simulation queue depth above 100 for 10m"
- alert: PolicyApprovalLatencyHigh
expr: histogram_quantile(0.95, sum(rate(policy_approval_latency_seconds_bucket[5m])) by (le)) > 30
for: 15m
labels:
severity: critical
service: policy
annotations:
summary: "Policy approval latency high"
description: "p95 approval latency above 30s for 15m"
- alert: PolicyPromotionFailureRate
expr: clamp_min(rate(policy_promotion_outcomes_total{outcome="failure"}[15m]), 0) / clamp_min(rate(policy_promotion_outcomes_total[15m]), 1) > 0.2
for: 10m
labels:
severity: critical
service: policy
annotations:
summary: "Policy promotion failure rate elevated"
description: "Failures exceed 20% of promotions over 15m"
- alert: PolicyPromotionStall
expr: rate(policy_promotion_outcomes_total{outcome="success"}[10m]) == 0 and sum(policy_simulation_queue_depth) > 0
for: 10m
labels:
severity: warning
service: policy
annotations:
summary: "Policy promotion stalled"
description: "No successful promotions while work is queued"

54
devops/telemetry/alerts/signals-alerts.yaml Normal file
View File

@@ -0,0 +1,54 @@
groups:
- name: signals-pipeline
rules:
- alert: SignalsScoringLatencyP95High
expr: histogram_quantile(0.95, sum(rate(signals_reachability_scoring_duration_seconds_bucket[5m])) by (le)) > 2
for: 10m
labels:
severity: warning
service: signals
annotations:
summary: "Signals scoring latency high (p95)"
description: "Reachability scoring p95 exceeds 2s for 10m"
- alert: SignalsCacheMissRateHigh
expr: |
clamp_min(rate(signals_cache_misses_total[5m]), 0)
/ clamp_min(rate(signals_cache_hits_total[5m]) + rate(signals_cache_misses_total[5m]), 1) > 0.3
for: 10m
labels:
severity: warning
service: signals
annotations:
summary: "Signals cache miss rate high"
description: "Cache miss ratio >30% over 10m; investigate Redis or key churn."
- alert: SignalsCacheDown
expr: signals_cache_available == 0
for: 2m
labels:
severity: critical
service: signals
annotations:
summary: "Signals cache unavailable"
description: "Redis cache reported unavailable for >2m"
- alert: SignalsSensorStaleness
expr: time() - max(signals_sensor_last_seen_timestamp_seconds) by (sensor) > 900
for: 5m
labels:
severity: warning
service: signals
annotations:
summary: "Signals sensor stale"
description: "No updates from sensor for >15 minutes"
- alert: SignalsIngestionErrorRate
expr: clamp_min(rate(signals_ingestion_failures_total[5m]), 0) / clamp_min(rate(signals_ingestion_total[5m]), 1) > 0.05
for: 5m
labels:
severity: critical
service: signals
annotations:
summary: "Signals ingestion failures elevated"
description: "Ingestion failure ratio above 5% over 5m"

62
devops/telemetry/alerts/triage-alerts.yaml Normal file
View File

@@ -0,0 +1,62 @@
groups:
- name: triage-ttfs
rules:
- alert: TriageTtfsFirstEvidenceP95High
expr: histogram_quantile(0.95, sum(rate(stellaops_ttfs_first_evidence_seconds_bucket[5m])) by (le)) > 1.5
for: 10m
labels:
severity: critical
service: triage
annotations:
summary: "TTFS first evidence p95 high"
description: "TTFS first-evidence p95 exceeds 1.5s for 10m (triage experience degraded)."
- alert: TriageTtfsSkeletonP95High
expr: histogram_quantile(0.95, sum(rate(stellaops_ttfs_skeleton_seconds_bucket[5m])) by (le)) > 0.2
for: 10m
labels:
severity: warning
service: triage
annotations:
summary: "TTFS skeleton p95 high"
description: "TTFS skeleton p95 exceeds 200ms for 10m."
- alert: TriageTtfsFullEvidenceP95High
expr: histogram_quantile(0.95, sum(rate(stellaops_ttfs_full_evidence_seconds_bucket[5m])) by (le)) > 1.5
for: 10m
labels:
severity: warning
service: triage
annotations:
summary: "TTFS full evidence p95 high"
description: "TTFS full-evidence p95 exceeds 1.5s for 10m."
- alert: TriageClicksToClosureMedianHigh
expr: histogram_quantile(0.50, sum(rate(stellaops_clicks_to_closure_bucket[5m])) by (le)) > 6
for: 15m
labels:
severity: warning
service: triage
annotations:
summary: "Clicks-to-closure median high"
description: "Median clicks-to-closure exceeds 6 for 15m."
- alert: TriageEvidenceCompletenessAvgLow
expr: (sum(rate(stellaops_evidence_completeness_score_sum[15m])) / clamp_min(sum(rate(stellaops_evidence_completeness_score_count[15m])), 1)) < 3.6
for: 30m
labels:
severity: warning
service: triage
annotations:
summary: "Evidence completeness below target"
description: "Average evidence completeness score below 3.6 (90%) for 30m."
- alert: TriageBudgetViolationRateHigh
expr: sum(rate(stellaops_performance_budget_violations_total[5m])) by (phase) > 0.05
for: 10m
labels:
severity: warning
service: triage
annotations:
summary: "Performance budget violations elevated"
description: "Performance budget violation rate exceeds 0.05/s for 10m."

90
devops/telemetry/otel-collector-config.yaml → devops/telemetry/collectors/otel-collector-config.yaml
View File

@@ -1,31 +1,31 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
tls:
cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set}
key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set}
client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set}
require_client_certificate: ${STELLAOPS_OTEL_REQUIRE_CLIENT_CERT:true}
http:
endpoint: 0.0.0.0:4318
tls:
cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set}
key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set}
client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set}
require_client_certificate: ${STELLAOPS_OTEL_REQUIRE_CLIENT_CERT:true}
processors:
attributes/tenant-tag:
actions:
- key: tenant.id
action: insert
value: ${STELLAOPS_TENANT_ID:unknown}
batch:
send_batch_size: 1024
timeout: 5s
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
tls:
cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set}
key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set}
client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set}
require_client_certificate: ${STELLAOPS_OTEL_REQUIRE_CLIENT_CERT:true}
http:
endpoint: 0.0.0.0:4318
tls:
cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set}
key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set}
client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set}
require_client_certificate: ${STELLAOPS_OTEL_REQUIRE_CLIENT_CERT:true}
processors:
attributes/tenant-tag:
actions:
- key: tenant.id
action: insert
value: ${STELLAOPS_TENANT_ID:unknown}
batch:
send_batch_size: 1024
timeout: 5s
exporters:
logging:
verbosity: normal
@@ -65,27 +65,27 @@ exporters:
enabled: true
queue_size: 1024
retry_on_failure: true
extensions:
health_check:
endpoint: ${STELLAOPS_OTEL_HEALTH_ENDPOINT:0.0.0.0:13133}
pprof:
endpoint: ${STELLAOPS_OTEL_PPROF_ENDPOINT:0.0.0.0:1777}
service:
telemetry:
logs:
level: ${STELLAOPS_OTEL_LOG_LEVEL:info}
extensions: [health_check, pprof]
pipelines:
extensions:
health_check:
endpoint: ${STELLAOPS_OTEL_HEALTH_ENDPOINT:0.0.0.0:13133}
pprof:
endpoint: ${STELLAOPS_OTEL_PPROF_ENDPOINT:0.0.0.0:1777}
service:
telemetry:
logs:
level: ${STELLAOPS_OTEL_LOG_LEVEL:info}
extensions: [health_check, pprof]
pipelines:
traces:
receivers: [otlp]
processors: [attributes/tenant-tag, batch]
exporters: [logging, otlphttp/tempo]
metrics:
receivers: [otlp]
processors: [attributes/tenant-tag, batch]
exporters: [logging, prometheus]
metrics:
receivers: [otlp]
processors: [attributes/tenant-tag, batch]
exporters: [logging, prometheus]
logs:
receivers: [otlp]
processors: [attributes/tenant-tag, batch]

536
devops/telemetry/dashboards/stella-ops-error-tracking.json Normal file
View File

@@ -0,0 +1,536 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": "${datasource}",
"enable": true,
"expr": "increase(stella_error_total[1m]) > 0",
"iconColor": "red",
"name": "Error Spikes",
"tagKeys": "error_type",
"titleFormat": "Error: {{error_type}}"
}
]
},
"description": "Stella Ops Release Orchestrator - Error Tracking",
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"id": null,
"iteration": 1737158400000,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"id": 1,
"panels": [],
"title": "Error Summary",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 10 }
]
}
},
"overrides": []
},
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(increase(stella_error_total[1h]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Errors (1h)",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 0.01 },
{ "color": "red", "value": 0.05 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(rate(stella_error_total[5m])) / sum(rate(stella_api_requests_total[5m]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Error Rate",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 5 }
]
}
},
"overrides": []
},
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(increase(stella_release_failed_total[1h]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Failed Releases (1h)",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 3 }
]
}
},
"overrides": []
},
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(increase(stella_gate_failed_total[1h]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Gate Failures (1h)",
"type": "stat"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
"id": 6,
"panels": [],
"title": "Error Trends",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "normal" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"id": 7,
"options": {
"legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(rate(stella_error_total[5m])) by (error_type)",
"legendFormat": "{{error_type}}",
"refId": "A"
}
],
"title": "Errors by Type",
"type": "timeseries"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "normal" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"id": 8,
"options": {
"legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(rate(stella_error_total{environment=~\"$environment\"}[5m])) by (component)",
"legendFormat": "{{component}}",
"refId": "A"
}
],
"title": "Errors by Component",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
"id": 9,
"panels": [],
"title": "Release Failures",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineWidth": 1,
"scaleDistribution": { "type": "linear" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
}
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
"id": 10,
"options": {
"barRadius": 0.1,
"barWidth": 0.8,
"groupWidth": 0.7,
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom" },
"orientation": "horizontal",
"showValue": "auto",
"stacking": "none",
"tooltip": { "mode": "single", "sort": "none" },
"xTickLabelRotation": 0,
"xTickLabelSpacing": 0
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "topk(10, sum(increase(stella_release_failed_total[24h])) by (failure_reason))",
"format": "table",
"instant": true,
"legendFormat": "{{failure_reason}}",
"refId": "A"
}
],
"title": "Top Failure Reasons (24h)",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": { "Time": true },
"indexByName": {},
"renameByName": { "Value": "Count", "failure_reason": "Reason" }
}
}
],
"type": "barchart"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "normal" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": [
{
"matcher": { "id": "byName", "options": "Failures" },
"properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }]
},
{
"matcher": { "id": "byName", "options": "Rollbacks" },
"properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }]
}
]
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
"id": 11,
"options": {
"legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(increase(stella_release_failed_total{environment=~\"$environment\"}[1h])) by (environment)",
"legendFormat": "{{environment}} Failures",
"refId": "A"
},
{
"expr": "sum(increase(stella_rollback_total{environment=~\"$environment\"}[1h])) by (environment)",
"legendFormat": "{{environment}} Rollbacks",
"refId": "B"
}
],
"title": "Failures & Rollbacks by Environment",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
"id": 12,
"panels": [],
"title": "Recent Errors",
"type": "row"
},
{
"datasource": "${loki_datasource}",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 },
"id": 13,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "{app=\"stella-ops\"} |= \"error\" | json | level=~\"error|fatal\"",
"legendFormat": "",
"refId": "A"
}
],
"title": "Error Logs",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 36,
"style": "dark",
"tags": ["stella-ops", "errors"],
"templating": {
"list": [
{
"current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
"hide": 0,
"includeAll": false,
"label": "Metrics",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": { "selected": false, "text": "Loki", "value": "Loki" },
"hide": 0,
"includeAll": false,
"label": "Logs",
"multi": false,
"name": "loki_datasource",
"options": [],
"query": "loki",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": { "selected": true, "text": "All", "value": "$__all" },
"datasource": "${datasource}",
"definition": "label_values(stella_error_total, environment)",
"hide": 0,
"includeAll": true,
"label": "Environment",
"multi": true,
"name": "environment",
"options": [],
"query": { "query": "label_values(stella_error_total, environment)", "refId": "StandardVariableQuery" },
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": { "from": "now-6h", "to": "now" },
"timepicker": {},
"timezone": "",
"title": "Stella Ops - Error Tracking",
"uid": "stella-ops-errors",
"version": 1,
"weekStart": ""
}

607
devops/telemetry/dashboards/stella-ops-performance.json Normal file
View File

@@ -0,0 +1,607 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Stella Ops Release Orchestrator - Performance Metrics",
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"id": null,
"iteration": 1737158400000,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"id": 1,
"panels": [],
"title": "System Performance",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 0.7 },
{ "color": "red", "value": 0.9 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
"id": 2,
"options": {
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "avg(stella_cpu_usage_ratio{component=\"orchestrator\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "CPU Usage",
"type": "gauge"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 0.7 },
{ "color": "red", "value": 0.9 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
"id": 3,
"options": {
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "avg(stella_memory_usage_ratio{component=\"orchestrator\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Memory Usage",
"type": "gauge"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 100 },
{ "color": "red", "value": 500 }
]
},
"unit": "ms"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["mean"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(stella_api_request_duration_seconds_bucket[5m])) by (le)) * 1000",
"legendFormat": "",
"refId": "A"
}
],
"title": "API Latency (p95)",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
]
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(rate(stella_api_requests_total[5m]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Request Rate",
"type": "stat"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
"id": 6,
"panels": [],
"title": "Gate Evaluation Performance",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"id": 7,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(stella_gate_evaluation_duration_seconds_bucket{gate_type=~\"$gate_type\"}[5m])) by (le, gate_type))",
"legendFormat": "{{gate_type}} p99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(stella_gate_evaluation_duration_seconds_bucket{gate_type=~\"$gate_type\"}[5m])) by (le, gate_type))",
"legendFormat": "{{gate_type}} p50",
"refId": "B"
}
],
"title": "Gate Evaluation Duration by Type",
"type": "timeseries"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"id": 8,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(rate(stella_gate_evaluations_total{gate_type=~\"$gate_type\"}[5m])) by (gate_type)",
"legendFormat": "{{gate_type}}",
"refId": "A"
}
],
"title": "Gate Evaluations per Second",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
"id": 9,
"panels": [],
"title": "Cache Performance",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.7 },
{ "color": "green", "value": 0.9 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 15 },
"id": 10,
"options": {
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(stella_cache_hits_total) / (sum(stella_cache_hits_total) + sum(stella_cache_misses_total))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Cache Hit Ratio",
"type": "gauge"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": [
{
"matcher": { "id": "byName", "options": "Hits" },
"properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }]
},
{
"matcher": { "id": "byName", "options": "Misses" },
"properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }]
}
]
},
"gridPos": { "h": 6, "w": 12, "x": 6, "y": 15 },
"id": 11,
"options": {
"legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(rate(stella_cache_hits_total[5m])) by (cache_name)",
"legendFormat": "{{cache_name}} Hits",
"refId": "A"
},
{
"expr": "sum(rate(stella_cache_misses_total[5m])) by (cache_name)",
"legendFormat": "{{cache_name}} Misses",
"refId": "B"
}
],
"title": "Cache Hits vs Misses",
"type": "timeseries"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 0.7 },
{ "color": "red", "value": 0.9 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 15 },
"id": 12,
"options": {
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "stella_cache_size_bytes / stella_cache_max_size_bytes",
"legendFormat": "",
"refId": "A"
}
],
"title": "Cache Utilization",
"type": "gauge"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 },
"id": 13,
"panels": [],
"title": "Database Performance",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "ms"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
"id": 14,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(stella_db_query_duration_seconds_bucket[5m])) by (le, query_type)) * 1000",
"legendFormat": "{{query_type}} p95",
"refId": "A"
}
],
"title": "Database Query Duration (p95)",
"type": "timeseries"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
"id": 15,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "stella_db_connections_active",
"legendFormat": "Active",
"refId": "A"
},
{
"expr": "stella_db_connections_idle",
"legendFormat": "Idle",
"refId": "B"
},
{
"expr": "stella_db_connections_max",
"legendFormat": "Max",
"refId": "C"
}
],
"title": "Database Connection Pool",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 36,
"style": "dark",
"tags": ["stella-ops", "performance"],
"templating": {
"list": [
{
"current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
"hide": 0,
"includeAll": false,
"label": "Data Source",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": { "selected": true, "text": "All", "value": "$__all" },
"datasource": "${datasource}",
"definition": "label_values(stella_gate_evaluation_duration_seconds_bucket, gate_type)",
"hide": 0,
"includeAll": true,
"label": "Gate Type",
"multi": true,
"name": "gate_type",
"options": [],
"query": { "query": "label_values(stella_gate_evaluation_duration_seconds_bucket, gate_type)", "refId": "StandardVariableQuery" },
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": { "from": "now-6h", "to": "now" },
"timepicker": {},
"timezone": "",
"title": "Stella Ops - Performance Metrics",
"uid": "stella-ops-performance",
"version": 1,
"weekStart": ""
}

566
devops/telemetry/dashboards/stella-ops-release-overview.json Normal file
View File

@@ -0,0 +1,566 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": "${datasource}",
"enable": true,
"expr": "stella_release_promotion_completed{environment=~\"$environment\"}",
"iconColor": "green",
"name": "Promotions",
"tagKeys": "version,environment",
"titleFormat": "Promotion to {{environment}}"
}
]
},
"description": "Stella Ops Release Orchestrator - Release Overview",
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"id": null,
"iteration": 1737158400000,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"id": 1,
"panels": [],
"title": "Release Summary",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
]
}
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "count(stella_release_active{environment=~\"$environment\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Active Releases",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 5 },
{ "color": "red", "value": 10 }
]
}
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "count(stella_release_pending_approval{environment=~\"$environment\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Pending Approvals",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(stella_release_success_total{environment=~\"$environment\"}) / sum(stella_release_total{environment=~\"$environment\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Success Rate (24h)",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 900 },
{ "color": "red", "value": 1800 }
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["mean"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(stella_release_duration_seconds_bucket{environment=~\"$environment\"}[24h])) by (le))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Median Release Time",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "green", "value": 1 }
]
}
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(stella_gate_passed_total{environment=~\"$environment\"}) / sum(stella_gate_evaluated_total{environment=~\"$environment\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Gate Pass Rate",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
"overrides": []
},
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(stella_rollback_total{environment=~\"$environment\"})",
"legendFormat": "",
"refId": "A"
}
],
"title": "Rollbacks (24h)",
"type": "stat"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
"id": 8,
"panels": [],
"title": "Release Activity",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"id": 9,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(rate(stella_release_total{environment=~\"$environment\"}[5m])) by (environment)",
"legendFormat": "{{environment}}",
"refId": "A"
}
],
"title": "Releases per Minute",
"type": "timeseries"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "normal" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "short"
},
"overrides": [
{
"matcher": { "id": "byName", "options": "Success" },
"properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }]
},
{
"matcher": { "id": "byName", "options": "Failed" },
"properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }]
}
]
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"id": 10,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(increase(stella_release_success_total{environment=~\"$environment\"}[1h]))",
"legendFormat": "Success",
"refId": "A"
},
{
"expr": "sum(increase(stella_release_failed_total{environment=~\"$environment\"}[1h]))",
"legendFormat": "Failed",
"refId": "B"
}
],
"title": "Release Outcomes (Hourly)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
"id": 11,
"panels": [],
"title": "Environment Health",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [
{ "options": { "0": { "color": "red", "index": 0, "text": "Down" } }, "type": "value" },
{ "options": { "1": { "color": "green", "index": 1, "text": "Up" } }, "type": "value" }
],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "green", "value": 1 }
]
}
},
"overrides": []
},
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 15 },
"id": 12,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "value_and_name"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "stella_environment_health{environment=~\"$environment\"}",
"legendFormat": "{{environment}}",
"refId": "A"
}
],
"title": "Environment Status",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "off" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{ "color": "green", "value": null }]
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 6, "w": 16, "x": 8, "y": 15 },
"id": 13,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right" },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(stella_release_duration_seconds_bucket{environment=~\"$environment\"}[5m])) by (le, environment))",
"legendFormat": "{{environment}} p95",
"refId": "A"
},
{
"expr": "histogram_quantile(0.50, sum(rate(stella_release_duration_seconds_bucket{environment=~\"$environment\"}[5m])) by (le, environment))",
"legendFormat": "{{environment}} p50",
"refId": "B"
}
],
"title": "Release Duration by Environment",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 36,
"style": "dark",
"tags": ["stella-ops", "releases"],
"templating": {
"list": [
{
"current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
"hide": 0,
"includeAll": false,
"label": "Data Source",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": { "selected": true, "text": "All", "value": "$__all" },
"datasource": "${datasource}",
"definition": "label_values(stella_release_total, environment)",
"hide": 0,
"includeAll": true,
"label": "Environment",
"multi": true,
"name": "environment",
"options": [],
"query": { "query": "label_values(stella_release_total, environment)", "refId": "StandardVariableQuery" },
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": { "from": "now-24h", "to": "now" },
"timepicker": {},
"timezone": "",
"title": "Stella Ops - Release Overview",
"uid": "stella-ops-releases",
"version": 1,
"weekStart": ""
}

541
devops/telemetry/dashboards/stella-ops-sla-monitoring.json Normal file
View File

@@ -0,0 +1,541 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": "${datasource}",
"enable": true,
"expr": "changes(stella_sla_breach_total[1m]) > 0",
"iconColor": "red",
"name": "SLA Breaches",
"tagKeys": "sla_name",
"titleFormat": "SLA Breach: {{sla_name}}"
}
]
},
"description": "Stella Ops Release Orchestrator - SLA Monitoring",
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"id": null,
"iteration": 1737158400000,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"id": 1,
"panels": [],
"title": "SLA Overview",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.99 },
{ "color": "green", "value": 0.999 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 },
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "1 - (sum(increase(stella_release_failed_total[30d])) / sum(increase(stella_release_total[30d])))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Release Success Rate (30d SLA)",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.99 },
{ "color": "green", "value": 0.999 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 },
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "avg_over_time(stella_api_availability[30d])",
"legendFormat": "",
"refId": "A"
}
],
"title": "API Availability (30d SLA)",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 300 },
{ "color": "red", "value": 600 }
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 },
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["mean"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(stella_release_duration_seconds_bucket[30d])) by (le))",
"legendFormat": "",
"refId": "A"
}
],
"title": "Release Time p95 (Target: <10m)",
"type": "stat"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
"overrides": []
},
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 },
"id": 5,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
"textMode": "auto"
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "sum(increase(stella_sla_breach_total[30d]))",
"legendFormat": "",
"refId": "A"
}
],
"title": "SLA Breaches (30d)",
"type": "stat"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 },
"id": 6,
"panels": [],
"title": "Error Budget",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 20 },
{ "color": "green", "value": 50 }
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 7 },
"id": 7,
"options": {
"orientation": "auto",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "((0.001 * sum(increase(stella_release_total[30d]))) - sum(increase(stella_release_failed_total[30d]))) / (0.001 * sum(increase(stella_release_total[30d]))) * 100",
"legendFormat": "",
"refId": "A"
}
],
"title": "Error Budget Remaining (99.9% SLA)",
"type": "gauge"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "line" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 0 }
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": { "h": 6, "w": 16, "x": 8, "y": 7 },
"id": 8,
"options": {
"legend": { "calcs": [], "displayMode": "list", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "(0.001 * sum(increase(stella_release_total[30d]))) - sum(increase(stella_release_failed_total[30d]))",
"legendFormat": "Remaining Budget (failures allowed)",
"refId": "A"
}
],
"title": "Error Budget Burn Rate",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 },
"id": 9,
"panels": [],
"title": "SLI Trends",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "line+area" }
},
"mappings": [],
"max": 1,
"min": 0.99,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "transparent", "value": 0.999 }
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
"id": 10,
"options": {
"legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "1 - (sum(rate(stella_release_failed_total[1h])) / sum(rate(stella_release_total[1h])))",
"legendFormat": "Success Rate",
"refId": "A"
}
],
"title": "Release Success Rate Over Time",
"type": "timeseries"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "palette-classic" },
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": { "type": "linear" },
"showPoints": "never",
"spanNulls": false,
"stacking": { "group": "A", "mode": "none" },
"thresholdsStyle": { "mode": "line+area" }
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "transparent", "value": null },
{ "color": "red", "value": 600 }
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
"id": 11,
"options": {
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" },
"tooltip": { "mode": "multi", "sort": "none" }
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(stella_release_duration_seconds_bucket[1h])) by (le))",
"legendFormat": "p95 Duration",
"refId": "A"
},
{
"expr": "histogram_quantile(0.99, sum(rate(stella_release_duration_seconds_bucket[1h])) by (le))",
"legendFormat": "p99 Duration",
"refId": "B"
}
],
"title": "Release Duration SLI",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 },
"id": 12,
"panels": [],
"title": "SLA by Environment",
"type": "row"
},
{
"datasource": "${datasource}",
"fieldConfig": {
"defaults": {
"color": { "mode": "thresholds" },
"custom": {
"align": "auto",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "yellow", "value": 0.99 },
{ "color": "green", "value": 0.999 }
]
}
},
"overrides": [
{
"matcher": { "id": "byName", "options": "Success Rate" },
"properties": [
{ "id": "unit", "value": "percentunit" },
{ "id": "custom.displayMode", "value": "color-background-solid" }
]
},
{
"matcher": { "id": "byName", "options": "Avg Duration" },
"properties": [{ "id": "unit", "value": "s" }]
}
]
},
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 23 },
"id": 13,
"options": {
"footer": { "fields": "", "reducer": ["sum"], "show": false },
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.0.0",
"targets": [
{
"expr": "1 - (sum(increase(stella_release_failed_total[7d])) by (environment) / sum(increase(stella_release_total[7d])) by (environment))",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
},
{
"expr": "sum(increase(stella_release_total[7d])) by (environment)",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "B"
},
{
"expr": "avg(rate(stella_release_duration_seconds_sum[7d]) / rate(stella_release_duration_seconds_count[7d])) by (environment)",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "C"
}
],
"title": "SLA by Environment (7d)",
"transformations": [
{
"id": "seriesToColumns",
"options": { "byField": "environment" }
},
{
"id": "organize",
"options": {
"excludeByName": { "Time 1": true, "Time 2": true, "Time 3": true },
"indexByName": {},
"renameByName": {
"Value #A": "Success Rate",
"Value #B": "Total Releases",
"Value #C": "Avg Duration",
"environment": "Environment"
}
}
}
],
"type": "table"
}
],
"refresh": "5m",
"schemaVersion": 36,
"style": "dark",
"tags": ["stella-ops", "sla"],
"templating": {
"list": [
{
"current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
"hide": 0,
"includeAll": false,
"label": "Data Source",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": { "from": "now-30d", "to": "now" },
"timepicker": {},
"timezone": "",
"title": "Stella Ops - SLA Monitoring",
"uid": "stella-ops-sla",
"version": 1,
"weekStart": ""
}

555
devops/telemetry/grafana/dashboards/attestation-metrics.json
View File

@@ -1,555 +0,0 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 0.9
},
{
"color": "green",
"value": 0.95
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": true,
"showThresholdMarkers": true
},
"pluginVersion": "10.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(stella_attestations_created_total) / (sum(stella_attestations_created_total) + sum(stella_attestations_failed_total))",
"refId": "A"
}
],
"title": "Attestation Completeness (Target: ≥95%)",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 30
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 9,
"x": 6,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": ["mean", "max"],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, rate(stella_ttfe_seconds_bucket[5m]))",
"legendFormat": "p95",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, rate(stella_ttfe_seconds_bucket[5m]))",
"legendFormat": "p50",
"refId": "B"
}
],
"title": "TTFE Distribution (Target: ≤30s)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "none",
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
},
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 9,
"x": 15,
"y": 0
},
"id": 3,
"options": {
"legend": {
"calcs": ["mean", "last"],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(stella_attestations_verified_total[5m])) / (sum(rate(stella_attestations_verified_total[5m])) + sum(rate(stella_attestations_failed_total[5m])))",
"legendFormat": "Success Rate",
"refId": "A"
}
],
"title": "Verification Success Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "none",
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
},
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "line"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 4,
"options": {
"legend": {
"calcs": ["sum"],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (environment, reason) (rate(stella_post_deploy_reversions_total[5m]))",
"legendFormat": "{{environment}}: {{reason}}",
"refId": "A"
}
],
"title": "Post-Deploy Reversions (Trend to Zero)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 12,
"y": 8
},
"id": 5,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": ["value"]
},
"pieType": "pie",
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (predicate_type) (stella_attestations_created_total)",
"legendFormat": "{{predicate_type}}",
"refId": "A"
}
],
"title": "Attestations by Type",
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "none",
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
},
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 6,
"x": 18,
"y": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(stella_attestations_failed_total{reason=\"stale_evidence\"})",
"legendFormat": "Stale Evidence Alerts",
"refId": "A"
}
],
"title": "Stale Evidence Alerts",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 38,
"style": "dark",
"tags": ["stellaops", "attestations", "security"],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"label": "Data Source",
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "StellaOps - Attestation Metrics",
"uid": "stellaops-attestations",
"version": 1,
"weekStart": ""
}

1016
devops/telemetry/grafana/dashboards/provcache-overview.json
View File

File diff suppressed because it is too large Load Diff

308
devops/telemetry/grafana/dashboards/stella-ops-p0-metrics.json
View File

@@ -1,308 +0,0 @@
{
"__comment": "Sprint: SPRINT_20260117_028_Telemetry_p0_metrics - P0 Product Metrics Dashboard",
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Time from fresh install to first successful verified promotion",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 14400 },
{ "color": "red", "value": 86400 }
]
},
"unit": "s"
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
"id": 1,
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": ["p90"],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"title": "Time to First Verified Release (P90)",
"type": "gauge",
"targets": [
{
"expr": "histogram_quantile(0.90, sum(rate(stella_time_to_first_verified_release_seconds_bucket{tenant=~\"$tenant\"}[24h])) by (le))",
"legendFormat": "P90",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Time from block decision to user viewing explanation",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 300 },
{ "color": "red", "value": 3600 }
]
},
"unit": "s"
}
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
"id": 2,
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": ["p90"],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"title": "Why Blocked Latency (P90)",
"type": "gauge",
"targets": [
{
"expr": "histogram_quantile(0.90, sum(rate(stella_why_blocked_latency_seconds_bucket{tenant=~\"$tenant\"}[24h])) by (le))",
"legendFormat": "P90",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Support minutes per tenant this month",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 30 },
{ "color": "red", "value": 60 }
]
},
"unit": "m"
}
},
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
"id": 3,
"options": {
"displayMode": "lcd",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showUnfilled": true
},
"title": "Support Burden (minutes/month)",
"type": "bargauge",
"targets": [
{
"expr": "sum by (tenant, category) (stella_support_burden_minutes_total{month=~\"$month\", tenant=~\"$tenant\"})",
"legendFormat": "{{tenant}} - {{category}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Determinism regression count by severity",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
},
"unit": "short"
}
},
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"title": "Determinism Regressions",
"type": "stat",
"targets": [
{
"expr": "sum by (severity) (stella_determinism_regressions_total{tenant=~\"$tenant\"})",
"legendFormat": "{{severity}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Time to first release heatmap over time",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
"id": 5,
"options": {
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"show": true,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false,
"unit": "s"
}
},
"title": "Time to First Release Distribution",
"type": "heatmap",
"targets": [
{
"expr": "sum(rate(stella_time_to_first_verified_release_seconds_bucket{tenant=~\"$tenant\"}[1h])) by (le)",
"format": "heatmap",
"legendFormat": "{{le}}",
"refId": "A"
}
]
}
],
"refresh": "30s",
"schemaVersion": 38,
"style": "dark",
"tags": ["stella-ops", "p0-metrics", "product"],
"templating": {
"list": [
{
"current": {},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"definition": "label_values(stella_time_to_first_verified_release_seconds_count, tenant)",
"hide": 0,
"includeAll": true,
"label": "Tenant",
"multi": true,
"name": "tenant",
"options": [],
"query": {
"query": "label_values(stella_time_to_first_verified_release_seconds_count, tenant)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"current": {
"selected": true,
"text": "2026-01",
"value": "2026-01"
},
"hide": 0,
"label": "Month",
"name": "month",
"options": [
{ "selected": true, "text": "2026-01", "value": "2026-01" },
{ "selected": false, "text": "2025-12", "value": "2025-12" }
],
"query": "2026-01,2025-12",
"skipUrlSync": false,
"type": "custom"
}
]
},
"time": {
"from": "now-7d",
"to": "now"
},
"timepicker": {},
"timezone": "utc",
"title": "Stella Ops P0 Product Metrics",
"uid": "stella-ops-p0-metrics",
"version": 1,
"weekStart": ""
}

36
devops/telemetry/storage/README.md
View File

@@ -1,36 +0,0 @@
# Telemetry Storage Stack
Configuration snippets for the default StellaOps observability backends used in
staging and production environments. The stack comprises:
- **Prometheus** for metrics (scraping the collector's Prometheus exporter)
- **Tempo** for traces (OTLP ingest via mTLS)
- **Loki** for logs (HTTP ingest with tenant isolation)
## Files
| Path | Description |
| ---- | ----------- |
| `prometheus.yaml` | Scrape configuration for the collector (mTLS + bearer token placeholder). |
| `tempo.yaml` | Tempo configuration with multitenancy enabled and local storage paths. |
| `loki.yaml` | Loki configuration enabling per-tenant overrides and boltdb-shipper storage. |
| `tenants/tempo-overrides.yaml` | Example tenant overrides for Tempo (retention, limits). |
| `tenants/loki-overrides.yaml` | Example tenant overrides for Loki (rate limits, retention). |
| `auth/` | Placeholder directory for Prometheus bearer token files (e.g., `token`). |
These configurations are referenced by the Docker Compose overlay
(`deploy/compose/docker-compose.telemetry-storage.yaml`) and the staging rollout documented in
`docs/modules/telemetry/operations/storage.md`. Adjust paths, credentials, and overrides before running in
connected environments. Place the Prometheus bearer token in `auth/token` when using the
Compose overlay (the directory contains a `.gitkeep` placeholder and is gitignored by default).
Run `python ops/devops/telemetry/validate_storage_stack.py` after editing any of these files to
ensure TLS, multitenancy, and override references remain intact.
## Security
- Both Tempo and Loki require mutual TLS.
- Prometheus uses mTLS plus a bearer token that should be minted by Authority.
- Update the overrides files to enforce per-tenant retention/ingestion limits.
For comprehensive deployment steps see `docs/modules/telemetry/operations/storage.md`.

0
devops/telemetry/storage/auth/.gitkeep
View File

19
devops/telemetry/storage/tenants/loki-overrides.yaml
View File

@@ -1,19 +0,0 @@
# Example Loki per-tenant overrides
# Adjust according to https://grafana.com/docs/loki/latest/configuration/#limits_config
stellaops-dev:
ingestion_rate_mb: 10
ingestion_burst_size_mb: 20
max_global_streams_per_user: 5000
retention_period: 168h
stellaops-stage:
ingestion_rate_mb: 20
ingestion_burst_size_mb: 40
max_global_streams_per_user: 10000
retention_period: 336h
__default__:
ingestion_rate_mb: 5
ingestion_burst_size_mb: 10
retention_period: 72h

16
devops/telemetry/storage/tenants/tempo-overrides.yaml
View File

@@ -1,16 +0,0 @@
# Example Tempo per-tenant overrides
# Consult https://grafana.com/docs/tempo/latest/configuration/#limits-configuration
# before applying in production.
stellaops-dev:
traces_per_second_limit: 100000
max_bytes_per_trace: 10485760
max_search_bytes_per_trace: 20971520
stellaops-stage:
traces_per_second_limit: 200000
max_bytes_per_trace: 20971520
__default__:
traces_per_second_limit: 50000
max_bytes_per_trace: 5242880

33
devops/telemetry/validation/README.md
View File

@@ -1,33 +0,0 @@
# Telemetry bundle verifier
Files:
- `verify-telemetry-bundle.sh`: offline verifier (checksums + optional JSON schema)
- `tests/sample-bundle/telemetry-bundle.json`: sample manifest
- `tests/sample-bundle/telemetry-bundle.sha256`: checksum list for sample bundle
- `tests/telemetry-bundle.tar`: deterministic sample bundle (ustar, mtime=0, owner/group 0)
- `tests/run-schema-tests.sh`: validates sample config against config schema
- `tests/ci-run.sh`: runs schema test + bundle verifier (use in CI)
Dependencies for full validation:
- `python` with `jsonschema` installed (`pip install jsonschema`)
- `tar`, `sha256sum`
Deterministic TAR flags used for sample bundle:
`tar --mtime=@0 --owner=0 --group=0 --numeric-owner --format=ustar`
Exit codes:
- 0 success
- 21 missing manifest/checksums
- 22 checksum mismatch
- 23 schema validation failed
- 64 usage error
Quick check:
```bash
./verify-telemetry-bundle.sh tests/telemetry-bundle.tar
```
CI suggestion:
```bash
ops/devops/telemetry/tests/ci-run.sh
```

77
devops/telemetry/validation/generate_dev_tls.sh
View File

@@ -1,77 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CERT_DIR="${SCRIPT_DIR}/../../deploy/telemetry/certs"
mkdir -p "${CERT_DIR}"
CA_KEY="${CERT_DIR}/ca.key"
CA_CRT="${CERT_DIR}/ca.crt"
COL_KEY="${CERT_DIR}/collector.key"
COL_CSR="${CERT_DIR}/collector.csr"
COL_CRT="${CERT_DIR}/collector.crt"
CLIENT_KEY="${CERT_DIR}/client.key"
CLIENT_CSR="${CERT_DIR}/client.csr"
CLIENT_CRT="${CERT_DIR}/client.crt"
echo "[*] Generating OpenTelemetry dev CA and certificates in ${CERT_DIR}"
# Root CA
if [[ ! -f "${CA_KEY}" ]]; then
openssl genrsa -out "${CA_KEY}" 4096 >/dev/null 2>&1
fi
openssl req -x509 -new -key "${CA_KEY}" -days 365 -sha256 \
-out "${CA_CRT}" -subj "/CN=StellaOps Dev Telemetry CA" \
-config <(cat <<'EOF'
[req]
distinguished_name = req_distinguished_name
prompt = no
[req_distinguished_name]
EOF
) >/dev/null 2>&1
# Collector certificate (server + client auth)
openssl req -new -nodes -newkey rsa:4096 \
-keyout "${COL_KEY}" \
-out "${COL_CSR}" \
-subj "/CN=stellaops-otel-collector" >/dev/null 2>&1
openssl x509 -req -in "${COL_CSR}" -CA "${CA_CRT}" -CAkey "${CA_KEY}" \
-CAcreateserial -out "${COL_CRT}" -days 365 -sha256 \
-extensions v3_req -extfile <(cat <<'EOF'
[v3_req]
subjectAltName = @alt_names
extendedKeyUsage = serverAuth, clientAuth
[alt_names]
DNS.1 = stellaops-otel-collector
DNS.2 = localhost
IP.1 = 127.0.0.1
EOF
) >/dev/null 2>&1
# Client certificate
openssl req -new -nodes -newkey rsa:4096 \
-keyout "${CLIENT_KEY}" \
-out "${CLIENT_CSR}" \
-subj "/CN=stellaops-otel-client" >/dev/null 2>&1
openssl x509 -req -in "${CLIENT_CSR}" -CA "${CA_CRT}" -CAkey "${CA_KEY}" \
-CAcreateserial -out "${CLIENT_CRT}" -days 365 -sha256 \
-extensions v3_req -extfile <(cat <<'EOF'
[v3_req]
extendedKeyUsage = clientAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = stellaops-otel-client
DNS.2 = localhost
IP.1 = 127.0.0.1
EOF
) >/dev/null 2>&1
rm -f "${COL_CSR}" "${CLIENT_CSR}"
rm -f "${CERT_DIR}/ca.srl"
echo "[✓] Certificates ready:"
ls -1 "${CERT_DIR}"

136
devops/telemetry/validation/package_offline_bundle.py
View File

@@ -1,136 +0,0 @@
#!/usr/bin/env python3
"""Package telemetry collector assets for offline/air-gapped installs.
Outputs a tarball containing the collector configuration, Compose overlay,
Helm defaults, and operator README. A SHA-256 checksum sidecar is emitted, and
optional Cosign signing can be enabled with --sign.
"""
from __future__ import annotations
import argparse
import hashlib
import os
import subprocess
import sys
import tarfile
from pathlib import Path
from typing import Iterable
REPO_ROOT = Path(__file__).resolve().parents[3]
DEFAULT_OUTPUT = REPO_ROOT / "out" / "telemetry" / "telemetry-offline-bundle.tar.gz"
BUNDLE_CONTENTS: tuple[Path, ...] = (
Path("deploy/telemetry/README.md"),
Path("deploy/telemetry/otel-collector-config.yaml"),
Path("deploy/telemetry/storage/README.md"),
Path("deploy/telemetry/storage/prometheus.yaml"),
Path("deploy/telemetry/storage/tempo.yaml"),
Path("deploy/telemetry/storage/loki.yaml"),
Path("deploy/telemetry/storage/tenants/tempo-overrides.yaml"),
Path("deploy/telemetry/storage/tenants/loki-overrides.yaml"),
Path("deploy/helm/stellaops/files/otel-collector-config.yaml"),
Path("deploy/helm/stellaops/values.yaml"),
Path("deploy/helm/stellaops/templates/otel-collector.yaml"),
Path("deploy/compose/docker-compose.telemetry.yaml"),
Path("deploy/compose/docker-compose.telemetry-storage.yaml"),
Path("docs/modules/telemetry/operations/collector.md"),
Path("docs/modules/telemetry/operations/storage.md"),
)
def compute_sha256(path: Path) -> str:
sha = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
sha.update(chunk)
return sha.hexdigest()
def validate_files(paths: Iterable[Path]) -> None:
missing = [str(p) for p in paths if not (REPO_ROOT / p).exists()]
if missing:
raise FileNotFoundError(f"Missing bundle artefacts: {', '.join(missing)}")
def create_bundle(output_path: Path) -> Path:
output_path.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(output_path, "w:gz") as tar:
for rel_path in BUNDLE_CONTENTS:
abs_path = REPO_ROOT / rel_path
tar.add(abs_path, arcname=str(rel_path))
return output_path
def write_checksum(bundle_path: Path) -> Path:
digest = compute_sha256(bundle_path)
sha_path = bundle_path.with_suffix(bundle_path.suffix + ".sha256")
sha_path.write_text(f"{digest} {bundle_path.name}\n", encoding="utf-8")
return sha_path
def cosign_sign(bundle_path: Path, key_ref: str | None, identity_token: str | None) -> None:
cmd = ["cosign", "sign-blob", "--yes", str(bundle_path)]
if key_ref:
cmd.extend(["--key", key_ref])
env = os.environ.copy()
if identity_token:
env["COSIGN_IDENTITY_TOKEN"] = identity_token
try:
subprocess.run(cmd, check=True, env=env)
except FileNotFoundError as exc:
raise RuntimeError("cosign not found on PATH; install cosign or omit --sign") from exc
except subprocess.CalledProcessError as exc:
raise RuntimeError(f"cosign sign-blob failed: {exc}") from exc
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--output",
type=Path,
default=DEFAULT_OUTPUT,
help=f"Output bundle path (default: {DEFAULT_OUTPUT})",
)
parser.add_argument(
"--sign",
action="store_true",
help="Sign the bundle using cosign (requires cosign on PATH)",
)
parser.add_argument(
"--cosign-key",
type=str,
default=os.environ.get("COSIGN_KEY_REF"),
help="Cosign key reference (file:..., azurekms://..., etc.)",
)
parser.add_argument(
"--identity-token",
type=str,
default=os.environ.get("COSIGN_IDENTITY_TOKEN"),
help="OIDC identity token for keyless signing",
)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
validate_files(BUNDLE_CONTENTS)
bundle_path = args.output.resolve()
print(f"[*] Creating telemetry bundle at {bundle_path}")
create_bundle(bundle_path)
sha_path = write_checksum(bundle_path)
print(f"[✓] SHA-256 written to {sha_path}")
if args.sign:
print("[*] Signing bundle with cosign")
cosign_sign(bundle_path, args.cosign_key, args.identity_token)
sig_path = bundle_path.with_suffix(bundle_path.suffix + ".sig")
if sig_path.exists():
print(f"[✓] Cosign signature written to {sig_path}")
else:
print("[!] Cosign completed but signature file not found (ensure cosign version >= 2.2)")
return 0
if __name__ == "__main__":
sys.exit(main())

197
devops/telemetry/validation/smoke_otel_collector.py
View File

@@ -1,197 +0,0 @@
#!/usr/bin/env python3
"""
Smoke test for the StellaOps OpenTelemetry Collector deployment.
The script sends sample traces, metrics, and logs over OTLP/HTTP with mutual TLS
and asserts that the collector accepted the payloads by checking its Prometheus
metrics endpoint.
"""
from __future__ import annotations
import argparse
import json
import ssl
import sys
import time
import urllib.request
from pathlib import Path
TRACE_PAYLOAD = {
"resourceSpans": [
{
"resource": {
"attributes": [
{"key": "service.name", "value": {"stringValue": "smoke-client"}},
{"key": "tenant.id", "value": {"stringValue": "dev"}},
]
},
"scopeSpans": [
{
"scope": {"name": "smoke-test"},
"spans": [
{
"traceId": "00000000000000000000000000000001",
"spanId": "0000000000000001",
"name": "smoke-span",
"kind": 1,
"startTimeUnixNano": "1730000000000000000",
"endTimeUnixNano": "1730000000500000000",
"status": {"code": 0},
}
],
}
],
}
]
}
METRIC_PAYLOAD = {
"resourceMetrics": [
{
"resource": {
"attributes": [
{"key": "service.name", "value": {"stringValue": "smoke-client"}},
{"key": "tenant.id", "value": {"stringValue": "dev"}},
]
},
"scopeMetrics": [
{
"scope": {"name": "smoke-test"},
"metrics": [
{
"name": "smoke_gauge",
"gauge": {
"dataPoints": [
{
"asDouble": 1.0,
"timeUnixNano": "1730000001000000000",
"attributes": [
{"key": "phase", "value": {"stringValue": "ingest"}}
],
}
]
},
}
],
}
],
}
]
}
LOG_PAYLOAD = {
"resourceLogs": [
{
"resource": {
"attributes": [
{"key": "service.name", "value": {"stringValue": "smoke-client"}},
{"key": "tenant.id", "value": {"stringValue": "dev"}},
]
},
"scopeLogs": [
{
"scope": {"name": "smoke-test"},
"logRecords": [
{
"timeUnixNano": "1730000002000000000",
"severityNumber": 9,
"severityText": "Info",
"body": {"stringValue": "StellaOps collector smoke log"},
}
],
}
],
}
]
}
def _load_context(ca: Path, cert: Path, key: Path) -> ssl.SSLContext:
context = ssl.create_default_context(cafile=str(ca))
context.check_hostname = False
context.verify_mode = ssl.CERT_REQUIRED
context.load_cert_chain(certfile=str(cert), keyfile=str(key))
return context
def _post_json(url: str, payload: dict, context: ssl.SSLContext) -> None:
data = json.dumps(payload).encode("utf-8")
request = urllib.request.Request(
url,
data=data,
headers={
"Content-Type": "application/json",
"User-Agent": "stellaops-otel-smoke/1.0",
},
method="POST",
)
with urllib.request.urlopen(request, context=context, timeout=10) as response:
if response.status // 100 != 2:
raise RuntimeError(f"{url} returned HTTP {response.status}")
def _fetch_metrics(url: str, context: ssl.SSLContext) -> str:
request = urllib.request.Request(
url,
headers={
"User-Agent": "stellaops-otel-smoke/1.0",
},
)
with urllib.request.urlopen(request, context=context, timeout=10) as response:
return response.read().decode("utf-8")
def _assert_counter(metrics: str, metric_name: str) -> None:
for line in metrics.splitlines():
if line.startswith(metric_name):
try:
_, value = line.split(" ")
if float(value) > 0:
return
except ValueError:
continue
raise AssertionError(f"{metric_name} not incremented")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--host", default="localhost", help="Collector host (default: %(default)s)")
parser.add_argument("--otlp-port", type=int, default=4318, help="OTLP/HTTP port")
parser.add_argument("--metrics-port", type=int, default=9464, help="Prometheus metrics port")
parser.add_argument("--health-port", type=int, default=13133, help="Health check port")
parser.add_argument("--ca", type=Path, default=Path("deploy/telemetry/certs/ca.crt"), help="CA certificate path")
parser.add_argument("--cert", type=Path, default=Path("deploy/telemetry/certs/client.crt"), help="Client certificate path")
parser.add_argument("--key", type=Path, default=Path("deploy/telemetry/certs/client.key"), help="Client key path")
args = parser.parse_args()
for path in (args.ca, args.cert, args.key):
if not path.exists():
print(f"[!] missing TLS material: {path}", file=sys.stderr)
return 1
context = _load_context(args.ca, args.cert, args.key)
otlp_base = f"https://{args.host}:{args.otlp_port}/v1"
print(f"[*] Sending OTLP traffic to {otlp_base}")
_post_json(f"{otlp_base}/traces", TRACE_PAYLOAD, context)
_post_json(f"{otlp_base}/metrics", METRIC_PAYLOAD, context)
_post_json(f"{otlp_base}/logs", LOG_PAYLOAD, context)
# Allow Prometheus exporter to update metrics
time.sleep(2)
metrics_url = f"https://{args.host}:{args.metrics_port}/metrics"
print(f"[*] Fetching collector metrics from {metrics_url}")
metrics = _fetch_metrics(metrics_url, context)
_assert_counter(metrics, "otelcol_receiver_accepted_spans")
_assert_counter(metrics, "otelcol_receiver_accepted_logs")
_assert_counter(metrics, "otelcol_receiver_accepted_metric_points")
print("[✓] Collector accepted traces, logs, and metrics.")
return 0
if __name__ == "__main__":
raise SystemExit(main())

232
devops/telemetry/validation/tenant_isolation_smoke.py
View File

@@ -1,232 +0,0 @@
#!/usr/bin/env python3
"""Tenant isolation smoke test for DEVOPS-OBS-50-002.
The script assumes the telemetry storage stack (Tempo + Loki) is running with
mutual TLS enabled and enforces `X-Scope-OrgID` multi-tenancy. It performs the
following checks:
1. Pushes a trace via the collector OTLP/HTTP endpoint and verifies it is
retrievable from Tempo when using the matching tenant header, but not when
querying as a different tenant.
2. Pushes a log entry to Loki with a tenant header and verifies it is only
visible to the matching tenant.
The goal is to provide a deterministic CI-friendly check that our storage
configuration preserves tenant isolation guard rails before promoting bundles.
"""
from __future__ import annotations
import argparse
import json
import ssl
import sys
import time
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
def _load_context(ca_file: Path, cert_file: Path, key_file: Path) -> ssl.SSLContext:
context = ssl.create_default_context(cafile=str(ca_file))
context.minimum_version = ssl.TLSVersion.TLSv1_2
context.check_hostname = False
context.load_cert_chain(certfile=str(cert_file), keyfile=str(key_file))
return context
def _post_json(url: str, payload: dict, context: ssl.SSLContext, headers: dict | None = None) -> None:
body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
request = urllib.request.Request(
url,
data=body,
method="POST",
headers={
"Content-Type": "application/json",
"User-Agent": "stellaops-tenant-smoke/1.0",
**(headers or {}),
},
)
with urllib.request.urlopen(request, context=context, timeout=10) as response:
status = response.status
if status // 100 != 2:
raise RuntimeError(f"POST {url} returned HTTP {status}")
def _get(url: str, context: ssl.SSLContext, headers: dict | None = None) -> tuple[int, str]:
request = urllib.request.Request(
url,
method="GET",
headers={
"User-Agent": "stellaops-tenant-smoke/1.0",
**(headers or {}),
},
)
try:
with urllib.request.urlopen(request, context=context, timeout=10) as response:
return response.status, response.read().decode("utf-8")
except urllib.error.HTTPError as exc: # type: ignore[attr-defined]
body = exc.read().decode("utf-8") if exc.fp else ""
return exc.code, body
def _payload_trace(trace_id: str, tenant: str) -> dict:
return {
"resourceSpans": [
{
"resource": {
"attributes": [
{"key": "service.name", "value": {"stringValue": "tenant-smoke"}},
{"key": "tenant.id", "value": {"stringValue": tenant}},
]
},
"scopeSpans": [
{
"scope": {"name": "tenant-smoke"},
"spans": [
{
"traceId": trace_id,
"spanId": "0000000000000001",
"name": "tenant-check",
"kind": 1,
"startTimeUnixNano": "1730500000000000000",
"endTimeUnixNano": "1730500000500000000",
"status": {"code": 0},
}
],
}
],
}
]
}
def _payload_log(ts_ns: int, tenant: str, marker: str) -> dict:
return {
"resourceLogs": [
{
"resource": {
"attributes": [
{"key": "service.name", "value": {"stringValue": "tenant-smoke"}},
{"key": "tenant.id", "value": {"stringValue": tenant}},
]
},
"scopeLogs": [
{
"scope": {"name": "tenant-smoke"},
"logRecords": [
{
"timeUnixNano": str(ts_ns),
"severityNumber": 9,
"severityText": "Info",
"body": {"stringValue": f"tenant={tenant} marker={marker}"},
}
],
}
],
}
]
}
def _assert_tenant_access(
tempo_url: str,
loki_url: str,
collector_url: str,
tenant: str,
other_tenant: str,
context: ssl.SSLContext,
) -> None:
trace_id = uuid.uuid4().hex + uuid.uuid4().hex[:16]
trace_payload = _payload_trace(trace_id, tenant)
_post_json(f"{collector_url}/traces", trace_payload, context)
log_marker = uuid.uuid4().hex[:12]
timestamp_ns = int(time.time() * 1_000_000_000)
log_payload = _payload_log(timestamp_ns, tenant, log_marker)
_post_json(f"{collector_url}/logs", log_payload, context)
# Allow background processing to flush to storage.
time.sleep(2)
tempo_headers = {"X-Scope-OrgID": tenant}
tempo_status, tempo_body = _get(f"{tempo_url}/api/traces/{trace_id}", context, headers=tempo_headers)
if tempo_status != 200:
raise AssertionError(f"Tempo returned HTTP {tempo_status} for tenant {tenant}: {tempo_body}")
if trace_id not in tempo_body:
raise AssertionError("Tempo response missing expected trace data")
other_status, _ = _get(
f"{tempo_url}/api/traces/{trace_id}", context, headers={"X-Scope-OrgID": other_tenant}
)
if other_status not in (401, 403, 404):
raise AssertionError(
f"Tempo should deny tenant {other_tenant}, received status {other_status}"
)
log_query = urllib.parse.urlencode({"query": "{app=\"tenant-smoke\"}"})
loki_status, loki_body = _get(
f"{loki_url}/loki/api/v1/query?{log_query}", context, headers={"X-Scope-OrgID": tenant}
)
if loki_status != 200:
raise AssertionError(f"Loki returned HTTP {loki_status} for tenant {tenant}: {loki_body}")
if log_marker not in loki_body:
raise AssertionError("Loki response missing expected log entry")
other_log_status, other_log_body = _get(
f"{loki_url}/loki/api/v1/query?{log_query}",
context,
headers={"X-Scope-OrgID": other_tenant},
)
if other_log_status == 200 and log_marker in other_log_body:
raise AssertionError("Loki returned tenant data to the wrong org")
if other_log_status not in (200, 401, 403):
raise AssertionError(
f"Unexpected Loki status when querying as {other_tenant}: {other_log_status}"
)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--collector", default="https://localhost:4318/v1", help="Collector OTLP base URL")
parser.add_argument("--tempo", default="https://localhost:3200", help="Tempo base URL")
parser.add_argument("--loki", default="https://localhost:3100", help="Loki base URL")
parser.add_argument("--tenant", default="dev", help="Primary tenant ID to test")
parser.add_argument("--other-tenant", default="stage", help="Secondary tenant expected to be denied")
parser.add_argument("--ca", type=Path, default=Path("deploy/telemetry/certs/ca.crt"), help="CA certificate path")
parser.add_argument(
"--cert", type=Path, default=Path("deploy/telemetry/certs/client.crt"), help="mTLS client certificate"
)
parser.add_argument(
"--key", type=Path, default=Path("deploy/telemetry/certs/client.key"), help="mTLS client key"
)
args = parser.parse_args()
for path in (args.ca, args.cert, args.key):
if not path.exists():
print(f"[!] missing TLS material: {path}", file=sys.stderr)
return 1
context = _load_context(args.ca, args.cert, args.key)
collector_base = args.collector.rstrip("/")
tempo_base = args.tempo.rstrip("/")
loki_base = args.loki.rstrip("/")
print(f"[*] Validating tenant isolation using tenant={args.tenant} and other={args.other_tenant}")
_assert_tenant_access(
tempo_base,
loki_base,
collector_base,
tenant=args.tenant,
other_tenant=args.other_tenant,
context=context,
)
print("[✓] Tempo and Loki enforce tenant isolation with mTLS + scoped headers.")
return 0
if __name__ == "__main__":
raise SystemExit(main())

7
devops/telemetry/validation/tests/ci-run.sh
View File

@@ -1,7 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../../" && pwd)"
SCHEMA="$ROOT/docs/modules/telemetry/schemas/telemetry-bundle.schema.json"
"$ROOT/ops/devops/telemetry/tests/run-schema-tests.sh"
TELEMETRY_BUNDLE_SCHEMA="$SCHEMA" "$ROOT/ops/devops/telemetry/verify-telemetry-bundle.sh" "$ROOT/ops/devops/telemetry/tests/telemetry-bundle.tar"

35
devops/telemetry/validation/tests/config-valid.json
View File

@@ -1,35 +0,0 @@
{
"schemaVersion": "1.0.0",
"hashAlgorithm": "sha256",
"profiles": [
{
"name": "default",
"description": "default profile",
"collectorVersion": "otelcol/1.0.0",
"cryptoProfile": "fips",
"sealedMode": false,
"allowlistedEndpoints": ["http://localhost:4318"],
"exporters": [
{
"type": "otlp",
"endpoint": "http://localhost:4318",
"protocol": "http",
"compression": "none",
"enabled": true
}
],
"redactionPolicyUri": "https://example.com/redaction-policy.json",
"sampling": {
"strategy": "traceidratio",
"seed": "0000000000000001",
"rules": [
{"match": "service.name == 'api'", "priority": 10, "sampleRate": 0.2}
]
},
"tenantRouting": {
"attribute": "tenant.id",
"quotasPerTenant": {"tenant-a": 1000}
}
}
]
}

9
devops/telemetry/validation/tests/make-sample.sh
View File

@@ -1,9 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../" && pwd)"
BUNDLE_DIR="$ROOT/tests/sample-bundle"
mkdir -p "$BUNDLE_DIR"
cp "$ROOT/tests/manifest-valid.json" "$BUNDLE_DIR/telemetry-bundle.json"
(cd "$BUNDLE_DIR" && sha256sum telemetry-bundle.json > telemetry-bundle.sha256)
tar --mtime=@0 --owner=0 --group=0 --numeric-owner --format=ustar -C "$BUNDLE_DIR" -cf "$ROOT/tests/telemetry-bundle.tar" telemetry-bundle.json telemetry-bundle.sha256
echo "Wrote sample bundle to $ROOT/tests/telemetry-bundle.tar"

26
devops/telemetry/validation/tests/manifest-valid.json
View File

@@ -1,26 +0,0 @@
{
"schemaVersion": "1.0.0",
"bundleId": "00000000-0000-0000-0000-000000000001",
"createdAt": "2025-12-01T00:00:00Z",
"profileHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"collectorVersion": "otelcol/1.0.0",
"sealedMode": true,
"redactionManifest": "redaction-manifest.json",
"manifestHashAlgorithm": "sha256",
"timeAnchor": {
"type": "rfc3161",
"value": "dummy-token"
},
"artifacts": [
{
"path": "logs.ndjson",
"sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"mediaType": "application/x-ndjson",
"size": 123
}
],
"dsseEnvelope": {
"hash": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
"location": "bundle.dsse.json"
}
}

19
devops/telemetry/validation/tests/run-schema-tests.sh
View File

@@ -1,19 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../../" && pwd)"
if ! command -v python >/dev/null 2>&1; then
echo "python not found" >&2; exit 127; fi
if ! python - <<'PY' >/dev/null 2>&1; then
import jsonschema
PY
then
echo "python jsonschema module not installed" >&2; exit 127; fi
python - <<'PY'
import json, pathlib
from jsonschema import validate
root = pathlib.Path('ops/devops/telemetry/tests')
config = json.loads((root / 'config-valid.json').read_text())
schema = json.loads(pathlib.Path('docs/modules/telemetry/schemas/telemetry-config.schema.json').read_text())
validate(config, schema)
print('telemetry-config schema ok')
PY

26
devops/telemetry/validation/tests/sample-bundle/telemetry-bundle.json
View File

@@ -1,26 +0,0 @@
{
"schemaVersion": "1.0.0",
"bundleId": "00000000-0000-0000-0000-000000000001",
"createdAt": "2025-12-01T00:00:00Z",
"profileHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"collectorVersion": "otelcol/1.0.0",
"sealedMode": true,
"redactionManifest": "redaction-manifest.json",
"manifestHashAlgorithm": "sha256",
"timeAnchor": {
"type": "rfc3161",
"value": "dummy-token"
},
"artifacts": [
{
"path": "logs.ndjson",
"sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"mediaType": "application/x-ndjson",
"size": 123
}
],
"dsseEnvelope": {
"hash": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc",
"location": "bundle.dsse.json"
}
}

1
devops/telemetry/validation/tests/sample-bundle/telemetry-bundle.sha256
View File

@@ -1 +0,0 @@
6e3fedbf183aece5dfa14a90ebce955e2887d36747c424e628dc2cc03bcb0ed3 telemetry-bundle.json

1
devops/telemetry/validation/tests/telemetry-bundle.sha256
View File

@@ -1 +0,0 @@
6e3fedbf183aece5dfa14a90ebce955e2887d36747c424e628dc2cc03bcb0ed3 ops/devops/telemetry/tests/manifest-valid.json

BIN
devops/telemetry/validation/tests/telemetry-bundle.tar
View File

Binary file not shown.

83
devops/telemetry/validation/validate_storage_stack.py
View File

@@ -1,83 +0,0 @@
#!/usr/bin/env python3
"""
Static validation for the telemetry storage stack configuration.
Checks the Prometheus, Tempo, and Loki configuration snippets to ensure:
- mutual TLS is enabled end-to-end
- tenant override files are referenced
- multitenancy flags are set
- retention/limit defaults exist for __default__ tenant entries
This script is intended to back `DEVOPS-OBS-50-002` and can run in CI
before publishing bundles or rolling out staging updates.
"""
from __future__ import annotations
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
PROMETHEUS_PATH = REPO_ROOT / "deploy/telemetry/storage/prometheus.yaml"
TEMPO_PATH = REPO_ROOT / "deploy/telemetry/storage/tempo.yaml"
LOKI_PATH = REPO_ROOT / "deploy/telemetry/storage/loki.yaml"
TEMPO_OVERRIDES_PATH = REPO_ROOT / "deploy/telemetry/storage/tenants/tempo-overrides.yaml"
LOKI_OVERRIDES_PATH = REPO_ROOT / "deploy/telemetry/storage/tenants/loki-overrides.yaml"
def read(path: Path) -> str:
if not path.exists():
raise FileNotFoundError(f"Required configuration file missing: {path}")
return path.read_text(encoding="utf-8")
def assert_contains(haystack: str, needle: str, path: Path) -> None:
if needle not in haystack:
raise AssertionError(f"{path} is missing required snippet: {needle!r}")
def validate_prometheus() -> None:
content = read(PROMETHEUS_PATH)
assert_contains(content, "tls_config:", PROMETHEUS_PATH)
assert_contains(content, "ca_file:", PROMETHEUS_PATH)
assert_contains(content, "cert_file:", PROMETHEUS_PATH)
assert_contains(content, "key_file:", PROMETHEUS_PATH)
assert_contains(content, "authorization:", PROMETHEUS_PATH)
assert_contains(content, "credentials_file:", PROMETHEUS_PATH)
def validate_tempo() -> None:
content = read(TEMPO_PATH)
assert_contains(content, "multitenancy_enabled: true", TEMPO_PATH)
assert_contains(content, "require_client_cert: true", TEMPO_PATH)
assert_contains(content, "per_tenant_override_config", TEMPO_PATH)
overrides = read(TEMPO_OVERRIDES_PATH)
assert_contains(overrides, "__default__", TEMPO_OVERRIDES_PATH)
assert_contains(overrides, "traces_per_second_limit", TEMPO_OVERRIDES_PATH)
assert_contains(overrides, "max_bytes_per_trace", TEMPO_OVERRIDES_PATH)
def validate_loki() -> None:
content = read(LOKI_PATH)
assert_contains(content, "auth_enabled: true", LOKI_PATH)
assert_contains(content, "per_tenant_override_config", LOKI_PATH)
overrides = read(LOKI_OVERRIDES_PATH)
assert_contains(overrides, "__default__", LOKI_OVERRIDES_PATH)
assert_contains(overrides, "retention_period", LOKI_OVERRIDES_PATH)
def main() -> int:
try:
validate_prometheus()
validate_tempo()
validate_loki()
except (AssertionError, FileNotFoundError) as exc:
print(f"[❌] telemetry storage validation failed: {exc}", file=sys.stderr)
return 1
print("[✓] telemetry storage configuration meets multi-tenant guard rails.")
return 0
if __name__ == "__main__":
sys.exit(main())

76
devops/telemetry/validation/verify-telemetry-bundle.sh
View File

@@ -1,76 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Minimal offline verifier for telemetry bundles (v1)
# Exits:
# 0 success
# 21 checksum/manifest missing
# 22 checksum mismatch
# 23 schema validation failed
BUNDLE=${1:-}
SCHEMA_PATH=${TELEMETRY_BUNDLE_SCHEMA:-}
if [[ -z "$BUNDLE" ]]; then
echo "Usage: $0 path/to/telemetry-bundle.tar" >&2
echo "Optional: set TELEMETRY_BUNDLE_SCHEMA=/abs/path/to/telemetry-bundle.schema.json" >&2
exit 64
fi
WORKDIR=$(mktemp -d)
cleanup() { rm -rf "$WORKDIR"; }
trap cleanup EXIT
tar --extract --file "$BUNDLE" --directory "$WORKDIR"
MANIFEST="$WORKDIR/telemetry-bundle.json"
HASHES="$WORKDIR/telemetry-bundle.sha256"
if [[ ! -f "$MANIFEST" || ! -f "$HASHES" ]]; then
echo "Missing manifest or checksum file." >&2
exit 21
fi
# Verify checksums
pushd "$WORKDIR" >/dev/null
if ! sha256sum --quiet --check telemetry-bundle.sha256; then
echo "Checksum mismatch." >&2
exit 22
fi
popd >/dev/null
# JSON schema validation (optional if jsonschema not present).
if command -v python >/dev/null 2>&1; then
SCHEMA_FILE="$SCHEMA_PATH"
if [[ -z "$SCHEMA_FILE" ]]; then
SCHEMA_DIR="$(cd "$(dirname "$0")/../../docs/modules/telemetry/schemas" 2>/dev/null || echo "")"
SCHEMA_FILE="$SCHEMA_DIR/telemetry-bundle.schema.json"
fi
if [[ -n "$SCHEMA_FILE" && -f "$SCHEMA_FILE" ]]; then
python - "$MANIFEST" "$SCHEMA_FILE" <<'PY'
import json, sys
from jsonschema import validate, Draft202012Validator
manifest_path = sys.argv[1]
schema_path = sys.argv[2]
with open(manifest_path, 'r', encoding='utf-8') as f:
manifest = json.load(f)
with open(schema_path, 'r', encoding='utf-8') as f:
schema = json.load(f)
Draft202012Validator.check_schema(schema)
validate(manifest, schema)
PY
if [[ $? -ne 0 ]]; then
echo "Schema validation failed." >&2
exit 23
fi
else
echo "Schema file not found ($SCHEMA_FILE); skipping validation." >&2
fi
else
echo "jsonschema validation skipped (requires python + jsonschema)." >&2
fi
echo "Telemetry bundle verified." >&2
exit 0