Merge pull request #12240 from fosterseth/make_prometheus_grafana21.1.0

Add prometheus and grafana make commands for local environment

9 files changed, 181 insertions, 37 deletions

diff --git a/.gitignore b/.gitignore
index 3d2c6c7ec0..a5d367158c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,7 +38,7 @@ awx/ui/build
 awx/ui/.env.local
 awx/ui/instrumented
 rsyslog.pid
-tools/prometheus/data
+tools/prometheus
 tools/docker-compose/ansible/awx_dump.sql
 tools/docker-compose/Dockerfile
 tools/docker-compose/_build
diff --git a/Makefile b/Makefile
index dd4a78f120..3263edfbca 100644
--- a/Makefile
+++ b/Makefile
@@ -529,7 +529,12 @@ docker-compose-cluster-elk: awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
 prometheus:
-	docker run -u0 --net=tools_default --link=`docker ps | egrep -o "tools_awx(_run)?_([^ ]+)?"`:awxweb --volume `pwd`/tools/prometheus:/prometheus --name prometheus -d -p 0.0.0.0:9090:9090 prom/prometheus --web.enable-lifecycle --config.file=/prometheus/prometheus.yml
+	docker volume create prometheus
+	docker run -d --rm --net=_sources_default --link=awx_1:awx1 --volume prometheus-storage:/prometheus --volume `pwd`/tools/prometheus:/etc/prometheus --name prometheus -p 9090:9090 prom/prometheus
+
+grafana:
+	docker volume create grafana
+	docker run -d --rm --net=_sources_default --volume grafana-storage:/var/lib/grafana --volume `pwd`/tools/grafana:/etc/grafana/provisioning --name grafana -p 3001:3000 grafana/grafana-enterprise
 
 docker-compose-container-group:
 	MINIKUBE_CONTAINER_GROUP=true make docker-compose
diff --git a/tools/docker-compose/README.md b/tools/docker-compose/README.md
index aca6a5b063..817563ce67 100644
--- a/tools/docker-compose/README.md
+++ b/tools/docker-compose/README.md
@@ -459,3 +459,20 @@ ansible-playbook tools/docker-compose/ansible/plumb_splunk.yml
 ```
 
 Once the playbook is done running Splunk should now be setup in your development environment. You can log into the admin console (see above for username/password) and click on "Searching and Reporting" in the left hand navigation. In the search box enter `source="http:tower_logging_collections"` and click search.
+
+
+### Prometheus and Grafana integration
+
+Prometheus is a metrics collecting tool, and we support prometheus formatted data at the `api/v2/metrics` endpoint.
+
+1. Change the `username` and `password` in `tools/prometheus/prometheus.yml`. You can also change the scrape interval.
+2. (optional) if you are in a clustered environment, you can change the target to `haproxy:8043` so that the incoming prometheus requests go through the load balancer. Leaving it set to `awx1` also works.
+3. run `make prometheus`
+4. navigate to `http://localhost:9090/targets` and check that the metrics endpoint State is Up.
+5. Click the Graph tab, start typing a metric name, or use the Open metrics explorer button to find a metric to display (next to `Execute` button)
+
+Prometheus can display basic graphs of your data, but it is minimal. Often Prometheus is paired with an app like Grafana for better visualization features.
+
+1. `make grafana` to run a local docker grafana instance.
+2. Navigate to `http://localhost:3001`. Sign in, using `admin` for both username and password.
+4. Now you can create a dashboard and add panels for whichever metrics you like.
diff --git a/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2 b/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2
index 1def668189..bcf91ac12c 100644
--- a/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2
+++ b/tools/docker-compose/ansible/roles/sources/templates/haproxy.cfg.j2
@@ -46,4 +46,3 @@ listen stats
     bind *:1936
     stats enable
     stats uri /
-
diff --git a/tools/grafana/dashboards/awx_dashboard.yml b/tools/grafana/dashboards/awx_dashboard.yml
new file mode 100644
index 0000000000..9692ffe1b7
--- /dev/null
+++ b/tools/grafana/dashboards/awx_dashboard.yml
@@ -0,0 +1,8 @@
+---
+apiVersion: 1
+
+providers:
+  - name: awx
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/provisioning/dashboards/demo_dashboard.json
diff --git a/tools/grafana/dashboards/demo_dashboard.json b/tools/grafana/dashboards/demo_dashboard.json
new file mode 100644
index 0000000000..0294fa4168
--- /dev/null
+++ b/tools/grafana/dashboards/demo_dashboard.json
@@ -0,0 +1,134 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "awx_status_total",
+          "refId": "A"
+        }
+      ],
+      "title": "job status",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "awx-demo",
+  "uid": "GISWZOXnk",
+  "version": 6,
+  "weekStart": ""
+}
diff --git a/tools/grafana/datasources/prometheus_source.yml b/tools/grafana/datasources/prometheus_source.yml
new file mode 100644
index 0000000000..99c11684c4
--- /dev/null
+++ b/tools/grafana/datasources/prometheus_source.yml
@@ -0,0 +1,10 @@
+---
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    isDefault: true
+    access: proxy
+    url: http://prometheus:9090
+    editable: true
diff --git a/tools/prometheus/.gitignore b/tools/prometheus/.gitignore
deleted file mode 100644
index 41da0ad48f..0000000000
--- a/tools/prometheus/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-./data
diff --git a/tools/prometheus/prometheus.yml b/tools/prometheus/prometheus.yml
index 9d2464cae1..52f40d2bb6 100644
--- a/tools/prometheus/prometheus.yml
+++ b/tools/prometheus/prometheus.yml
@@ -1,46 +1,18 @@
 ---
-# prometheus.yml
-# my global config
 global:
   scrape_interval: 15s  # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-  evaluation_interval: 15s  # Evaluate rules every 15 seconds. The default is every 1 minute.
-  # scrape_timeout is set to the global default (10s).
 
-# Alertmanager configuration
-alerting:
-  alertmanagers:
-    - static_configs:
-        - targets:
-        # - alertmanager:9093
-
-# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
-rule_files:
-# - "first_rules.yml"
-# - "second_rules.yml"
-
-
-# A scrape configuration containing exactly one endpoint to scrape:
-# Here it's Prometheus itself.
 scrape_configs:
-  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
-  - job_name: 'prometheus'
-    # metrics_path defaults to '/metrics'
-    # scheme defaults to 'http'.
-    static_configs:
-      - targets: ['127.0.0.1:9090']
-
   - job_name: 'awx'
+    static_configs:
+      - targets: ['awx1:8043']  # or haproxy:8043 in cluster env
     tls_config:
       insecure_skip_verify: true
     metrics_path: /api/v2/metrics
     scrape_interval: 5s
-    scheme: http
+    scheme: https
     params:
       format: ['txt']
     basic_auth:
-      username: admin
-      password: password
-    # bearer_token: oauth-token
-    static_configs:
-      - targets:
-          - awxweb:8013
+      username: awxuser  # change this
+      password: password  # change this