{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" }, { "datasource": "${datasource}", "enable": true, "expr": "increase(stella_error_total[1m]) > 0", "iconColor": "red", "name": "Error Spikes", "tagKeys": "error_type", "titleFormat": "Error: {{error_type}}" } ] }, "description": "Stella Ops Release Orchestrator - Error Tracking", "editable": true, "gnetId": null, "graphTooltip": 1, "id": null, "iteration": 1737158400000, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [], "title": "Error Summary", "type": "row" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 10 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.0.0", "targets": [ { "expr": "sum(increase(stella_error_total[1h]))", "legendFormat": "", "refId": "A" } ], "title": "Errors (1h)", "type": "stat" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.0.0", "targets": [ { "expr": "sum(rate(stella_error_total[5m])) / sum(rate(stella_api_requests_total[5m]))", "legendFormat": "", "refId": "A" } ], "title": "Error Rate", "type": "stat" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.0.0", "targets": [ { "expr": "sum(increase(stella_release_failed_total[1h]))", "legendFormat": "", "refId": "A" } ], "title": "Failed Releases (1h)", "type": "stat" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, "id": 5, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.0.0", "targets": [ { "expr": "sum(increase(stella_gate_failed_total[1h]))", "legendFormat": "", "refId": "A" } ], "title": "Gate Failures (1h)", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 6, "panels": [], "title": "Error Trends", "type": "row" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 7, "options": { "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.0.0", "targets": [ { "expr": "sum(rate(stella_error_total[5m])) by (error_type)", "legendFormat": "{{error_type}}", "refId": "A" } ], "title": "Errors by Type", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 8, "options": { "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.0.0", "targets": [ { "expr": "sum(rate(stella_error_total{environment=~\"$environment\"}[5m])) by (component)", "legendFormat": "{{component}}", "refId": "A" } ], "title": "Errors by Component", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 9, "panels": [], "title": "Release Failures", "type": "row" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 10, "options": { "barRadius": 0.1, "barWidth": 0.8, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "orientation": "horizontal", "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, "pluginVersion": "9.0.0", "targets": [ { "expr": "topk(10, sum(increase(stella_release_failed_total[24h])) by (failure_reason))", "format": "table", "instant": true, "legendFormat": "{{failure_reason}}", "refId": "A" } ], "title": "Top Failure Reasons (24h)", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": { "Value": "Count", "failure_reason": "Reason" } } } ], "type": "barchart" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Failures" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, { "matcher": { "id": "byName", "options": "Rollbacks" }, "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "id": 11, "options": { "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.0.0", "targets": [ { "expr": "sum(increase(stella_release_failed_total{environment=~\"$environment\"}[1h])) by (environment)", "legendFormat": "{{environment}} Failures", "refId": "A" }, { "expr": "sum(increase(stella_rollback_total{environment=~\"$environment\"}[1h])) by (environment)", "legendFormat": "{{environment}} Rollbacks", "refId": "B" } ], "title": "Failures & Rollbacks by Environment", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 12, "panels": [], "title": "Recent Errors", "type": "row" }, { "datasource": "${loki_datasource}", "fieldConfig": { "defaults": {}, "overrides": [] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 }, "id": 13, "options": { "dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": true, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true }, "pluginVersion": "9.0.0", "targets": [ { "expr": "{app=\"stella-ops\"} |= \"error\" | json | level=~\"error|fatal\"", "legendFormat": "", "refId": "A" } ], "title": "Error Logs", "type": "logs" } ], "refresh": "30s", "schemaVersion": 36, "style": "dark", "tags": ["stella-ops", "errors"], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, "hide": 0, "includeAll": false, "label": "Metrics", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": { "selected": false, "text": "Loki", "value": "Loki" }, "hide": 0, "includeAll": false, "label": "Logs", "multi": false, "name": "loki_datasource", "options": [], "query": "loki", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": "${datasource}", "definition": "label_values(stella_error_total, environment)", "hide": 0, "includeAll": true, "label": "Environment", "multi": true, "name": "environment", "options": [], "query": { "query": "label_values(stella_error_total, environment)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Stella Ops - Error Tracking", "uid": "stella-ops-errors", "version": 1, "weekStart": "" }