Merge ~jacekn/prometheus-charm:refresh into prometheus-charm:master

[Jacek Nykis]

Add script to refresh external rules

[Junien F (axino)]

inline comments

review: Needs Fixing

[Junien F (axino)]

+1

review: Approve

diff --git a/config.yaml b/config.yaml
index 4f4a2bc..0d1e4d9 100644
--- a/config.yaml
+++ b/config.yaml
@@ -109,6 +109,11 @@ options:
         Plain text private SSH key used to for git authentication.
         Also see "rules-repo" config option
     default: ""
+  rules-update-notify-email:
+    type: string
+    description: |
+        Email address that will receive cronned rule update failure emails
+    default: ""
   external_url:
     default: ""
     type: string
diff --git a/reactive/prometheus.py b/reactive/prometheus.py
index 0fc50ba..9e6f2b2 100644
--- a/reactive/prometheus.py
+++ b/reactive/prometheus.py
@@ -22,6 +22,8 @@ PROMETHEUS_YML_TMPL = 'prometheus.yml.j2'
 PROMETHEUS_DEF_TMPL = 'etc_default_prometheus.j2'
 PROMETHEUS_RULES_CRON = '/etc/cron.d/juju-prometheus-rules-refresh'
 PROMETHEUS_SSH_KEYFILE = '/root/.ssh/prometheus_git_key'
+REFRESH_RULES_SCRIPT = '/usr/local/bin/refresh_rules.sh'
+REFRESH_RULES_SCRIPT_TMPL = 'refresh_rules.sh.j2'
 
 
 def templates_changed(tmpl_list):
@@ -75,28 +77,39 @@ def configure_rules():
         set_state('prometheus.do-reconfig-yml')
 
     if config.get('rules-repo', False):
-        cron_ssh_command = ''
-        cli_ssh_command = ''
+        ssh_command = ''
         if config.get('rules-update-ssh-key', False):
             host.mkdir(os.path.dirname(PROMETHEUS_SSH_KEYFILE), perms=0o700)
             with open(PROMETHEUS_SSH_KEYFILE, 'w') as f:
                 f.write(config['rules-update-ssh-key'])
             os.chmod(PROMETHEUS_SSH_KEYFILE, 0o600)
-            cli_ssh_command = 'ssh -i {}'.format(PROMETHEUS_SSH_KEYFILE)
-            cron_ssh_command = 'GIT_SSH_COMMAND="{}"'.format(cli_ssh_command)
+            ssh_command = 'ssh -i {}'.format(PROMETHEUS_SSH_KEYFILE)
 
         rules_path = get_paths()['external_rules_path']
         if os.path.exists(rules_path):
-            cmd = ['git', '-C', rules_path, 'pull']
+            subprocess.check_call(['git', '-C', rules_path, 'pull'],
+                                  env={'GIT_SSH_COMMAND': ssh_command})
         else:
             # New repo, store key during initial clone
-            cli_ssh_command += ' -o StrictHostKeyChecking=no'
-            cmd = ['git', 'clone', config['rules-repo'], rules_path]
-        subprocess.check_call(cmd, env={'GIT_SSH_COMMAND': cli_ssh_command})
+            subprocess.check_call(['git', 'clone', config['rules-repo'], rules_path],
+                                  env={'GIT_SSH_COMMAND': ssh_command + ' -o StrictHostKeyChecking=no'})
+
+        options = {
+            'prometheus_yml': get_paths()['prometheus_yml'],
+            'external_rules_path': get_paths()['external_rules_path'],
+            'promtool': get_paths()['promtool'],
+            'ssh_command': ssh_command,
+            'rules_update_notify_email': config.get('rules-update-notify-email', False),
+        }
+        render(source=REFRESH_RULES_SCRIPT_TMPL,
+               target=REFRESH_RULES_SCRIPT,
+               context=options,
+               perms=0o755,
+               )
 
         schedule = config.get('rules-update-schedule', '*/10 * * * *')
         with open(PROMETHEUS_RULES_CRON, 'w') as f:
-            f.write('{} root {} git -C {} pull\n'.format(schedule, cron_ssh_command, rules_path))
+            f.write('{} root {} &>/dev/null\n'.format(schedule, REFRESH_RULES_SCRIPT))
         set_state('prometheus.do-reconfig-yml')
     remove_state('prometheus.do-reconfig-rules')
 
@@ -168,6 +181,8 @@ def install_packages():
                                       'revert install_method option to previous value')
         return
 
+    if not os.path.exists('/usr/sbin/sendmail'):
+        fetch.apt_install('sendmail-bin')  # Needed for cronjob email notifications
     if source == 'snap':
         snap.install(pkgname, channel=channel, force_dangerous=False)
         if config.get('prometheus_registration_listen', False):
@@ -401,7 +416,7 @@ def check_reconfig_prometheus():
     install_opts = ('install_sources', 'install_keys', 'snap_channel',
                     'prometheus_registration_listen')
     def_change_opts = ('daemon-args', 'external_url')
-    rules_change_opts = ('rules-repo', 'rules-update-schedule', 'rules-update-ssh-key')
+    rules_change_opts = ('rules-repo', 'rules-update-schedule', 'rules-update-ssh-key', 'rules-update-notify-email')
     yml_change_opts = set(config.keys()) - set(install_opts + def_change_opts + rules_change_opts)
     if any(config.changed(opt) for opt in install_opts):
         set_state('prometheus.do-install')
@@ -413,10 +428,6 @@ def check_reconfig_prometheus():
         set_state('prometheus.do-reconfig-yml')
     if any((
         data_changed('prometheus.target_jobs', target_jobs),
-        templates_changed([PROMETHEUS_YML_TMPL]),
-    )):
-        set_state('prometheus.do-reconfig-yml')
-    if any((
         data_changed('prometheus.scrape_jobs', scrape_jobs),
         templates_changed([PROMETHEUS_YML_TMPL]),
     )):
@@ -426,6 +437,8 @@ def check_reconfig_prometheus():
         templates_changed([PROMETHEUS_DEF_TMPL]),
     )):
         set_state('prometheus.do-reconfig-def')
+    if templates_changed([REFRESH_RULES_SCRIPT_TMPL]):
+        set_state('prometheus.do-reconfig-rules')
     # Cron can update rules on disk, update-status hook can trigger config update
     if kv.get('prometheus.rule_files') != get_rule_files():
         kv.set('prometheus.rule_files', get_rule_files())
diff --git a/templates/refresh_rules.sh.j2 b/templates/refresh_rules.sh.j2
 new file mode 100644
index 0000000..c880cc0
--- /dev/null
+++ b/templates/refresh_rules.sh.j2
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+REPO_DIR={{ external_rules_path }}
+PROMTOOL={{ promtool }}
+PROM_YML={{ prometheus_yml }}
+GIT_SSH_COMMAND={{ cron_ssh_command }}
+if [ -n "${GIT_SSH_COMMAND}" ]; then
+    export GIT_SSH_COMMAND=${GIT_SSH_COMMAND}
+fi
+
+function notify_by_email {
+{% if rules_update_notify_email %}
+    /usr/sbin/sendmail -t << EOF
+To: {{ rules_update_notify_email }}
+From: {{ rules_update_notify_email }}
+Subject: Prometheus rule update failed on ${HOSTNAME}
+Automatic rules refresh on ${HOSTNAME} failed!
+
+Error: $1
+
+Please investigate
+EOF
+{% else %}
+echo "No email set, skipping notification"
+{% endif %}
+}
+
+cd $REPO_DIR
+echo "Running pre-update check"
+${PROMTOOL} check-config ${PROM_YML}
+if [ $? -eq 0 ]; then
+    git tag -f rules_live && git fetch && git merge --quiet --ff-only FETCH_HEAD
+    if [ $? -ne 0 ]; then
+        echo "Problem merging new rules"
+        notify_by_email "Problem merging new rules"
+        exit 1
+    fi
+else
+    echo "Pre-check failed, aborted refresh"
+    notify_by_email "Pre-check failed, aborted refresh"
+    exit 1
+fi
+
+echo "Running post-update check"
+${PROMTOOL} check-config ${PROM_YML}
+if [ $? -eq 0 ]; then
+    killall -HUP prometheus
+    git tag -f rules_live
+    echo "Update and reload complete"
+else
+    git reset --hard rules_live
+    echo "New rules did not pass verification, reverted"
+    notify_by_email "New rules did not pass verification, reverted"
+    exit 1
+fi