Merge ~peter-sabaini/charm-sudo-pair:add-logging into ~sudo-pair-charmers/charm-sudo-pair:master

[Peter Sabaini]

Pagerduty alerting, action logging

Ability to have pagerduty alerts triggered on auto-approve. Revamp logging, add logging for remove-sudopair action. Use https proxy to contact the PD events endpoint if set in model config.

[Giuseppe Petralia (peppepetra)]

Small comment on the pagerduty_proxy that may be removed in favor of the juju model-config if any.

Comments in line.

Other than that looks good to me.

review: Needs Fixing

[Paul Goins (vultaire)]

I agree with Giuseppe. I think we should pull the proxy from the env's JUJU_CHARM_HTTPS_PROXY variable to avoid proliferating proxy settings. Other than that I'm +1.

review: Needs Fixing

[Peter Sabaini (peter-sabaini)]

Thanks -- proxy config updated as requested. I've resubmitted the branch as there was some code reorg interim.

[Celia Wang (ziyiwang)]

LGTM.

[Paul Goins (vultaire)]

Sorry for the delay here - It looks like this is close to if not identical to https://code.launchpad.net/~peter-sabaini/charm-sudo-pair/+git/sudo-pair-charm/+merge/377884, except that it might be based off newer code. I don't see any changes re: proxy variable use; this code is still using pagerduty_proxy rather than e.g. JUJU_CHARM_HTTPS_PROXY.

review: Needs Fixing

Unmerged commits

55e2099... by Peter Sabaini on 2020-01-21

Pagerduty alerting, action logging

- Ability to have pagerduty alerts triggered on auto-approve

- Revamp logging, add logging for remove-sudopair action

diff --git a/actions/actions.py b/actions/actions.py
index ccf3ffc..9ff1b7f 100755
--- a/actions/actions.py
+++ b/actions/actions.py
@@ -13,21 +13,31 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import logging
 import os
+import subprocess
 import sys
 
 from charmhelpers.core.hookenv import action_fail, action_set
 
-
 sys.path.append("lib")
 
 import libsudopair  # NOQA
 
 
+logger = logging.getLogger('sudopair')
+logger.setLevel(logging.INFO)
+handler = logging.handlers.SysLogHandler(address='/dev/log', facility=logging.handlers.SysLogHandler.LOG_AUTH)
+logger.addHandler(handler)
+
+
 def remove():
     """Remove sudo-pair config and binaries."""
     sph = libsudopair.SudoPairHelper()
     sph.deconfigure()
+    msg = "WARNING removed sudo-pair via remove-sudopair action"
+    logger.warning(msg)
+    subprocess.run(["/usr/bin/pagerdutyevent.py", msg])
     action_set({"message": "Successfully removed sudo-pair config and binaries"})
 
 
diff --git a/config.yaml b/config.yaml
index 795269b..9eca76c 100644
--- a/config.yaml
+++ b/config.yaml
@@ -19,3 +19,17 @@ options:
     type: boolean
     default: true
     description: "If true, auto approval is permitted."
+  pagerduty_key:
+    type: string
+    default: ''
+    description: "If set, a pagerduty event will be triggered upon auto-approving"
+  pagerduty_proxy:
+    type: string
+    default: ''
+    description: "If set, will use as a proxy to reach the pagerduty API"
+  pagerduty_context:
+    type: string
+    default: ''
+    description: "Prefix to add to pagerduty events"
+
+
diff --git a/files/pagerdutyevent.py b/files/pagerdutyevent.py
 new file mode 100644
index 0000000..e154e04
--- /dev/null
+++ b/files/pagerdutyevent.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import argparse
+import configparser
+import json
+import requests
+import logging
+import logging.handlers
+
+
+logger = logging.getLogger('sudopair')
+logger.setLevel(logging.INFO)
+handler = logging.handlers.SysLogHandler(address='/dev/log', facility=logging.handlers.SysLogHandler.LOG_AUTH)
+logger.addHandler(handler)
+
+
+def args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("summary")
+    return parser.parse_args()
+
+
+def trigger_incident(pd_key, summary, source, proxy):
+
+    header = {
+        "Content-Type": "application/json"
+    }
+
+    payload = {
+        "routing_key": pd_key,
+        "event_action": "trigger",
+        "payload": {
+            "summary": summary,
+            "source": source,
+            "severity": "info"
+        }
+    }
+
+    response = requests.post('https://events.pagerduty.com/v2/enqueue',
+                             data=json.dumps(payload),
+                             proxies={'https': proxy},
+                             headers=header)
+
+    if response.json()["status"] == "success":
+        logger.info("Triggered alert with key {}".format(response.json()['dedup_key']))
+    else:
+        logger.warning("Failed to send pagerduty alert: {}".format(response.text))
+
+
+def get_conf():
+    config = configparser.ConfigParser()
+    config.read("/etc/sudo_pair.pagerduty.ini")
+    sec = config['DEFAULT']
+    return sec['pagerduty_key'], sec['pagerduty_source'], sec['pagerduty_proxy']
+
+
+def main():
+    pd_key, source, proxy = get_conf()
+    trigger_incident(pd_key, args().summary, source, proxy)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/lib/libsudopair.py b/lib/libsudopair.py
index 05d1f9d..4c01bb2 100644
--- a/lib/libsudopair.py
+++ b/lib/libsudopair.py
@@ -64,6 +64,9 @@ class SudoPairHelper(object):
         self.sudoers_perms = 0o440
         self.sudo_conf_perms = 0o644
         self.sudo_approve_perms = 0o755
+        self.pagerduty_conf_path = '/etc/sudo_pair.pagerduty.ini'
+        self.pagerduty_path = '/usr/bin/pagerdutyevent.py'
+        self.pagerduty_perms = 0o755
 
     def get_config(self):
         """Return config as a dict."""
@@ -74,8 +77,9 @@ class SudoPairHelper(object):
             'socket_dir': self.socket_dir,
             'gids_enforced': group_names_to_group_ids(self.charm_config['groups_enforced']),
             'gids_exempted': group_names_to_group_ids(self.charm_config['groups_exempted']),
+            'pagerduty_key': self.charm_config.get('pagerduty_key'),
+            'pagerduty_proxy': self.charm_config.get('pagerduty_proxy'),
         }
-
         config.update(self.charm_config)
         return config
 
@@ -117,6 +121,10 @@ class SudoPairHelper(object):
         sudoers_file = os.path.join(hookenv.charm_dir(), 'files', 'sudoers')
         copy_file(sudoers_file, self.sudoers_path, self.owner, self.group, self.sudoers_perms)
 
+    def copy_pagerduty(self):
+        pd_file = os.path.join(hookenv.charm_dir(), 'files', 'pagerdutyevent.py')
+        copy_file(pd_file, self.pagerduty_path, self.owner, self.group, self.pagerduty_perms)
+
     def render_sudo_approve(self):
         """Render sudo-approve file."""
         hookenv.log("Rendering sudo_approve.tmpl to {}".format(self.binary_path))
@@ -131,6 +139,14 @@ class SudoPairHelper(object):
                                      self.get_config(), perms=0o440, owner=self.owner, group=self.group)
         return None
 
+    def render_pagerduty_conf(self):
+        pd_source = '{}-{}'.format(self.charm_config.get('pagerduty_context', 'juju'), hookenv.principal_unit())
+        cfg = self.get_config()
+        cfg['pagerduty_source'] = pd_source
+        hookenv.log("Rendering pagerduty configuration to {}".format(self.pagerduty_conf_path))
+        return templating.render('sudo_pair.pagerduty.tmpl', self.pagerduty_conf_path, cfg,
+                                 owner=self.owner, group=self.group)
+
     def deconfigure(self):
         """Remove sudo-pair configuration."""
         paths = [
diff --git a/reactive/sudo_pair.py b/reactive/sudo_pair.py
index 974dd66..bce23ca 100644
--- a/reactive/sudo_pair.py
+++ b/reactive/sudo_pair.py
@@ -26,6 +26,10 @@ def install_sudo_pair():
     # Add "Defaults log_output to /etc/sudoers
     sph.copy_sudoers()
 
+    # Add pagerduty script and config
+    sph.copy_pagerduty()
+    sph.render_pagerduty_conf()
+
     # If there are cmds to bypass sudo pairing create file unders sudoers.d
     sph.render_bypass_cmds()
 
diff --git a/templates/sudo_approve.tmpl b/templates/sudo_approve.tmpl
index 7164b5a..49665d3 100755
--- a/templates/sudo_approve.tmpl
+++ b/templates/sudo_approve.tmpl
@@ -88,7 +88,7 @@ main() {
     declare -r username
 
     declare log_line
-    log_line="$(date "+[%b %d %H:%M:%S] WARNING: ${username} approved is own sudo session.")"
+    log_line="WARNING: ${username} approved own sudo session."
     declare -r log_line
 
     if [[ "${uid}" -eq "${ruid}" ]]; then
@@ -97,7 +97,10 @@ main() {
         exit 1
         {% else %}
         echo "You are approving your own session. The incident will be logged."
-        echo ${log_line} >> /var/log/sudo_pair.log
+        logger -p auth.warn $log_line
+        {% if pagerduty_key %}
+          /usr/bin/pagerdutyevent.py "$log_line"
+        {% endif %}
         {% endif %}
     fi
 
diff --git a/templates/sudo_pair.pagerduty.tmpl b/templates/sudo_pair.pagerduty.tmpl
 new file mode 100644
index 0000000..d670f23
--- /dev/null
+++ b/templates/sudo_pair.pagerduty.tmpl
@@ -0,0 +1,5 @@
+[DEFAULT]
+pagerduty_key: {{ pagerduty_key }}
+pagerduty_proxy: {{ pagerduty_proxy }}
+pagerduty_source: {{ pagerduty_source }}
+
diff --git a/tests/unit/test_libsudopair.py b/tests/unit/test_libsudopair.py
index 3a598af..9ddff50 100644
--- a/tests/unit/test_libsudopair.py
+++ b/tests/unit/test_libsudopair.py
@@ -131,7 +131,7 @@ class TestSudoPairHelper():
     def test_render_sudo_approve(self, sph, tmpdir):
         """Check that sudo_approve file is rendered correctly."""
         # Auto Approve true
-        expected_content = 'echo ${log_line} >> /var/log/sudo_pair.log'
+        expected_content = 'logger -p auth.warn $log_line'
         socket_dir = tmpdir.join('/var/run/sudo_pair')
         expected_content_socket_dir = 'declare -r SUDO_SOCKET_PATH="{}"'.format(socket_dir)
         content = sph.render_sudo_approve()