summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--tools/grafana/alerting/alerts.yml8
-rw-r--r--tools/grafana/dashboards/demo_dashboard.json423
2 files changed, 281 insertions, 150 deletions
diff --git a/tools/grafana/alerting/alerts.yml b/tools/grafana/alerting/alerts.yml
index 92325ce78c..a6679e2989 100644
--- a/tools/grafana/alerting/alerts.yml
+++ b/tools/grafana/alerting/alerts.yml
@@ -254,10 +254,10 @@ groups:
intervalMs: 1000
maxDataPoints: 43200
reducer: last
- refId: mean_redis_queue_size
+ refId: last_redis_queue_size
type: reduce
queryType: ""
- refId: mean_redis_queue_size
+ refId: last_redis_queue_size
relativeTimeRange:
from: 0
to: 0
@@ -273,7 +273,7 @@ groups:
type: and
query:
params:
- - mean_redis_queue_size
+ - last_redis_queue_size
reducer:
params: []
type: avg
@@ -282,7 +282,7 @@ groups:
name: Expression
type: __expr__
uid: __expr__
- expression: '($mean_redis_queue_size > ($mean_event_insertion_rate * 120))'
+ expression: '($last_redis_queue_size > ($mean_event_insertion_rate * 120))'
hide: false
intervalMs: 1000
maxDataPoints: 43200
diff --git a/tools/grafana/dashboards/demo_dashboard.json b/tools/grafana/dashboards/demo_dashboard.json
index 5f35787e9d..3589794499 100644
--- a/tools/grafana/dashboards/demo_dashboard.json
+++ b/tools/grafana/dashboards/demo_dashboard.json
@@ -24,7 +24,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
- "id": 1,
+ "id": 2,
"links": [],
"liveNow": false,
"panels": [
@@ -73,8 +73,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -83,10 +82,56 @@
]
}
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "failed"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "semi-dark-red",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "running"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "dark-green",
+ "mode": "fixed"
+ }
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "canceled"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "super-light-yellow",
+ "mode": "fixed"
+ }
+ }
+ ]
+ }
+ ]
},
"gridPos": {
- "h": 8,
+ "h": 6,
"w": 12,
"x": 0,
"y": 0
@@ -110,7 +155,10 @@
"type": "prometheus",
"uid": "awx_prometheus"
},
+ "editorMode": "code",
"expr": "awx_status_total",
+ "legendFormat": "{{status}}",
+ "range": true,
"refId": "A"
}
],
@@ -134,7 +182,7 @@
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
- "fillOpacity": 32,
+ "fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
@@ -162,60 +210,24 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "{instance=\"awx1:8013\", job=\"awx\", node=\"awx_1\"}"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "blue",
- "mode": "fixed"
- }
+ "color": "green"
},
{
- "id": "displayName",
- "value": "Job Events Processed/Minute"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "{__name__=\"callback_receiver_events_queue_size_redis\", instance=\"awx1:8013\", job=\"awx\", node=\"awx_1\"}"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "semi-dark-red",
- "mode": "fixed"
- }
- },
- {
- "id": "displayName",
- "value": "Redis Queue Size"
+ "color": "red",
+ "value": 80
}
]
}
- ]
+ },
+ "overrides": []
},
"gridPos": {
- "h": 8,
+ "h": 6,
"w": 12,
"x": 12,
"y": 0
},
- "id": 24,
+ "id": 27,
"options": {
"legend": {
"calcs": [],
@@ -235,25 +247,13 @@
"uid": "awx_prometheus"
},
"editorMode": "builder",
- "expr": "callback_receiver_events_queue_size_redis",
- "legendFormat": "__auto",
+ "expr": "awx_instance_remaining_capacity",
+ "legendFormat": "remaining_capacity_{{hostname}}",
"range": true,
"refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "awx_prometheus"
- },
- "editorMode": "code",
- "expr": "irate(callback_receiver_events_insert_db[1m])*60",
- "hide": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "B"
}
],
- "title": "Redis Queue Size vs. Job Events Processed/Minute",
+ "title": "Remaining Instance Capacity",
"type": "timeseries"
},
{
@@ -301,8 +301,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -317,7 +316,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 8
+ "y": 6
},
"id": 20,
"options": {
@@ -365,19 +364,16 @@
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
- "fillOpacity": 18,
+ "fillOpacity": 32,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
- "lineInterpolation": "smooth",
- "lineStyle": {
- "fill": "solid"
- },
+ "lineInterpolation": "linear",
"lineWidth": 1,
- "pointSize": 6,
+ "pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
@@ -396,12 +392,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
+ "color": "green"
}
]
}
@@ -410,19 +401,38 @@
{
"matcher": {
"id": "byName",
- "options": "{__name__=\"callback_receiver_event_processing_avg_seconds\", instance=\"awx1:8013\", job=\"awx\", node=\"awx_1\"}"
+ "options": "{instance=\"awx1:8013\", job=\"awx\", node=\"awx_1\"}"
},
"properties": [
{
"id": "color",
"value": {
- "fixedColor": "orange",
+ "fixedColor": "blue",
"mode": "fixed"
}
},
{
"id": "displayName",
- "value": "Avg Job Event Processing Lag Time"
+ "value": "Job Events Processed/Minute"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "{__name__=\"callback_receiver_events_queue_size_redis\", instance=\"awx1:8013\", job=\"awx\", node=\"awx_1\"}"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "semi-dark-red",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "displayName",
+ "value": "Redis Queue Size"
}
]
}
@@ -432,9 +442,9 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 8
+ "y": 6
},
- "id": 26,
+ "id": 24,
"options": {
"legend": {
"calcs": [],
@@ -454,13 +464,25 @@
"uid": "awx_prometheus"
},
"editorMode": "builder",
- "expr": "callback_receiver_event_processing_avg_seconds",
+ "expr": "callback_receiver_events_queue_size_redis",
"legendFormat": "__auto",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "awx_prometheus"
+ },
+ "editorMode": "code",
+ "expr": "irate(callback_receiver_events_insert_db[1m])*60",
+ "hide": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Avg Job Event Processing Lag Time",
+ "title": "Redis Queue Size vs. Job Events Processed/Minute",
"type": "timeseries"
},
{
@@ -508,8 +530,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -551,7 +572,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 16
+ "y": 14
},
"id": 12,
"options": {
@@ -611,7 +632,6 @@
"type": "prometheus",
"uid": "awx_prometheus"
},
- "description": "",
"fieldConfig": {
"defaults": {
"color": {
@@ -624,16 +644,19 @@
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
- "fillOpacity": 0,
+ "fillOpacity": 18,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
- "lineInterpolation": "linear",
+ "lineInterpolation": "smooth",
+ "lineStyle": {
+ "fill": "solid"
+ },
"lineWidth": 1,
- "pointSize": 5,
+ "pointSize": 6,
"scaleDistribution": {
"type": "linear"
},
@@ -652,8 +675,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -662,15 +684,35 @@
]
}
},
- "overrides": []
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "{__name__=\"callback_receiver_event_processing_avg_seconds\", instance=\"awx1:8013\", job=\"awx\", node=\"awx_1\"}"
+ },
+ "properties": [
+ {
+ "id": "color",
+ "value": {
+ "fixedColor": "orange",
+ "mode": "fixed"
+ }
+ },
+ {
+ "id": "displayName",
+ "value": "Avg Job Event Processing Lag Time"
+ }
+ ]
+ }
+ ]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
- "y": 16
+ "y": 14
},
- "id": 16,
+ "id": 26,
"options": {
"legend": {
"calcs": [],
@@ -690,45 +732,13 @@
"uid": "awx_prometheus"
},
"editorMode": "builder",
- "expr": "dependency_manager__schedule_seconds",
+ "expr": "callback_receiver_event_processing_avg_seconds",
"legendFormat": "__auto",
"range": true,
"refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "awx_prometheus"
- },
- "editorMode": "builder",
- "expr": "dependency_manager_generate_dependencies_seconds",
- "hide": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "awx_prometheus"
- },
- "editorMode": "builder",
- "expr": "dependency_manager_get_tasks_seconds",
- "hide": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "C"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "awx_prometheus"
- },
- "hide": false,
- "refId": "D"
}
],
- "title": "Dependency Manager Timings",
+ "title": "Avg Job Event Processing Lag Time",
"type": "timeseries"
},
{
@@ -776,8 +786,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -792,7 +801,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 24
+ "y": 22
},
"id": 10,
"options": {
@@ -876,6 +885,7 @@
"type": "prometheus",
"uid": "awx_prometheus"
},
+ "description": "",
"fieldConfig": {
"defaults": {
"color": {
@@ -916,8 +926,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -932,9 +941,9 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 24
+ "y": 22
},
- "id": 18,
+ "id": 16,
"options": {
"legend": {
"calcs": [],
@@ -954,7 +963,7 @@
"uid": "awx_prometheus"
},
"editorMode": "builder",
- "expr": "workflow_manager__schedule_seconds",
+ "expr": "dependency_manager__schedule_seconds",
"legendFormat": "__auto",
"range": true,
"refId": "A"
@@ -964,11 +973,35 @@
"type": "prometheus",
"uid": "awx_prometheus"
},
+ "editorMode": "builder",
+ "expr": "dependency_manager_generate_dependencies_seconds",
"hide": false,
+ "legendFormat": "__auto",
+ "range": true,
"refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "awx_prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "dependency_manager_get_tasks_seconds",
+ "hide": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "awx_prometheus"
+ },
+ "hide": false,
+ "refId": "D"
}
],
- "title": "Workflow Manager Timings",
+ "title": "Dependency Manager Timings",
"type": "timeseries"
},
{
@@ -1016,8 +1049,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -1032,7 +1064,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 32
+ "y": 30
},
"id": 14,
"options": {
@@ -1062,9 +1094,108 @@
],
"title": "Database",
"type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "awx_prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "id": 18,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "awx_prometheus"
+ },
+ "editorMode": "builder",
+ "expr": "workflow_manager__schedule_seconds",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "awx_prometheus"
+ },
+ "hide": false,
+ "refId": "B"
+ }
+ ],
+ "title": "Workflow Manager Timings",
+ "type": "timeseries"
}
],
- "refresh": false,
+ "refresh": "5s",
"schemaVersion": 37,
"style": "dark",
"tags": [],
@@ -1079,6 +1210,6 @@
"timezone": "",
"title": "awx-demo",
"uid": "GISWZOXnk",
- "version": 9,
+ "version": 2,
"weekStart": ""
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-03 07:29:50 +0100