Merge lp:~jacekn/mojo/mojo-specs into lp:mojo/mojo-specs

[Jacek Nykis]

Improve prometheus verification script. Add symlinks to allow upgrade-charm testing for prometheus-alertmanager, prometheus-blackbox-exporter, prometheus-pushgateway and prometheus-snmp-exporter

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Laurent Sesquès (sajoupa)]

+1

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 110

[lp:~jacekn/mojo/mojo-specs]

110. By Jacek Nykis on 2018-07-05

Improve prometheus verification script. Add symlinks to allow upgrade-charm testing for prometheus-alertmanager, prometheus-blackbox-exporter, prometheus-pushgateway and prometheus-snmp-exporter

Reviewed-on: https://code.launchpad.net/~jacekn/mojo/mojo-specs/+merge/349005
Reviewed-by: Laurent Sesques <email address hidden>

=== added symlink 'charm-testing/prometheus-alertmanager'
=== target is u'prometheus2'
=== added symlink 'charm-testing/prometheus-blackbox-exporter'
=== target is u'prometheus2'
=== added symlink 'charm-testing/prometheus-pushgateway'
=== target is u'prometheus2'
=== added symlink 'charm-testing/prometheus-snmp-exporter'
=== target is u'prometheus2'
=== added file 'charm-testing/prometheus2/helpers'
--- charm-testing/prometheus2/helpers	1970-01-01 00:00:00 +0000
+++ charm-testing/prometheus2/helpers	2018-07-05 14:12:29 +0000
@@ -0,0 +1,17 @@
+function get_units() {
+    service=${1}
+    INSTANCES=$(juju status --format=yaml ${service} 2>&1 | sed -nr '/^ {8}public-address/s/.*://p')
+
+    if [ -z "${INSTANCES}" ]; then
+        echo "No ${service} instances found"
+        exit 1
+    fi
+    echo ${INSTANCES}
+}
+
+function fail() {
+    echo "####################"
+    echo "# E2E Check Failed #"
+    echo "####################"
+    exit 1
+}

=== modified file 'charm-testing/prometheus2/post-deploy'
--- charm-testing/prometheus2/post-deploy	2018-06-28 12:27:42 +0000
+++ charm-testing/prometheus2/post-deploy	2018-07-05 14:12:29 +0000
@@ -1,35 +1,96 @@
 #!/bin/bash
 
-PROMETHEUS=$(juju status --format=yaml prometheus2 2>&1 | sed -nr '/^ {8}public-address/s/.*://p')
-
-if [ -z "${PROMETHEUS}" ]; then
-    echo "No prometheus2 instances"
-    exit 1
-fi
-
-for UNIT in ${PROMETHEUS}; do
-
-    echo "Calling unit to ensure it returns: 'up' metrics"
-    echo 'curl -f -s -S -v http://${UNIT}:9090/api/v1/query?query=up'
+. helpers
+
+##################################
+# prometheus2 verification
+##################################
+for UNIT in $(get_units prometheus2); do
+    echo -e "####\n# Calling unit to ensure it returns: 'up' metrics"
     curl -f -s -S -v http://${UNIT}:9090/api/v1/query?query=up
     if [ $? -ne 0 ]; then
-        echo "####################"
-        echo "# E2E Check Failed #"
-        echo "####################"
-        exit 1
+        fail
     fi
-    echo "Calling unit to ensure telegraf targets were registered correctly"
-    echo 'curl -f -s -S -g http://${UNIT}:9090/api/v1/query?query=up{job=\"telegraf\"}|grep telegraf'
+
+    echo -e "\n####\n# Calling unit to ensure telegraf targets were registered correctly"
     curl -f -s -S -g http://${UNIT}:9090/api/v1/query?query=up{job=\"telegraf\"}|grep telegraf
     if [ $? -ne 0 ]; then
-        echo "####################"
-        echo "# E2E Check Failed #"
-        echo "####################"
-        exit 1
-    fi
-done
-
-# If we are here it means no checks failed
+        fail
+    fi
+
+    echo -e "\n####\n# Calling unit to ensure pushgateway was registered correctly"
+    curl -f -s -S -g http://${UNIT}:9090/api/v1/query?query=up{job=\"prometheus-pushgateway\"}|grep prometheus-pushgateway
+    if [ $? -ne 0 ]; then
+        fail
+    fi
+
+    echo -e "\n####\n# Calling unit to ensure alertmanager was registered correctly"
+    curl -f -s -S -g http://${UNIT}:9090/api/v1/query?query=prometheus_notifications_alertmanagers_discovered==1|grep value
+    if [ $? -ne 0 ]; then
+        fail
+    fi
+done
+
+##################################
+# alertmanager
+##################################
+for UNIT in $(get_units prometheus-alertmanager); do
+    echo -e "####\n# Checking alertmanager"
+    curl -f -s -S -g http://${UNIT}:9093/metrics|grep alertmanager_build_info
+    if [ $? -ne 0 ]; then
+        fail
+    fi
+done
+
+##################################
+# pushgateway
+##################################
+for UNIT in $(get_units prometheus-pushgateway); do
+    echo -e "####\n# Checking pushgateway"
+    curl -f -s -S -g http://${UNIT}:9091/metrics|grep pushgateway_build_info
+    if [ $? -ne 0 ]; then
+        fail
+    fi
+done
+
+##################################
+# snmp-exporter
+##################################
+for UNIT in $(get_units prometheus-snmp-exporter); do
+    echo -e "####\n# Checking snmp-exporter"
+    curl -f -s -S -g http://${UNIT}:9116/metrics|grep snmp_exporter_build_info
+    if [ $? -ne 0 ]; then
+        fail
+    fi
+done
+
+##################################
+# blackbox-exporter
+##################################
+for UNIT in $(get_units prometheus-blackbox-exporter); do
+    echo -e "####\n# Checking blackbox-exporter"
+    curl -f -s -S -g http://${UNIT}:9115/metrics|grep blackbox_exporter_build_info
+    if [ $? -ne 0 ]; then
+        fail
+    fi
+done
+
+##################################
+# grafana
+##################################
+for UNIT in $(get_units grafana); do
+    echo -e "####\n# Checking grafana"
+    curl -f -s -S -g http://${UNIT}:3000/metrics|grep grafana_info
+    if [ $? -ne 0 ]; then
+        fail
+    fi
+    curl -f -s -S -g http://${UNIT}:3000/|grep "<title>Grafana</title>"
+    if [ $? -ne 0 ]; then
+        fail
+    fi
+done
+
+# If we are here it means all checks passed
 echo "#####################"
 echo "# E2E Checks Passed #"
 echo "#####################"