Merge ~canonical-bootstack/charm-openstack-service-checks:feature/contrail-service-checks into ~canonical-bootstack/charm-openstack-service-checks:master

[Jeremy Lounder]

Add Nagios check for Juniper Contrail Analytics alarms

Based on a new charm originally proposed by npochet into Juniper's contrail-charms: https://github.com/Juniper/contrail-charms/pull/70/files
The check also parses the /analytics/alarms URL into Nagios alert format.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Alvaro Uria (aluria)]

keystone.yaml will be removed per conversation, so use the already existing nagios.novarc config file. I've also suggested a minor change on the contrail_vip property, as I was recommended in the past by the OpenStack charmers.

We may want to validate that the contrail_vip is valid and make "check_contrail_alarms.py -H <vip>" work.

[Alvaro Uria (aluria)]

Sample of the output generated by the check: https://pastebin.canonical.com/p/RqG7KbGZgg/

[Jeremy Lounder (jldev)]

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change cannot be self approved, setting status to needs review.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 3bc2b50c4960827288234d14d30fdcb02eb45af8

diff --git a/config.yaml b/config.yaml
index 86ade7d..e571370 100644
--- a/config.yaml
+++ b/config.yaml
@@ -104,3 +104,7 @@ options:
       URL to use with check_http if there is a Swift endpoint.  Default is '/', but it's possible to add extra params,
       e.g. '/v3 -e Unauthorized -d x-openstack-request-id' or a different url, e.g. '/healthcheck'.  Mitaka Swift
       typically needs '/healthcheck'.
+  contrail_analytics_vip:
+    type: string
+    default: ''
+    description: The VIP used for Contrail Analytics. Leave blank to disable Contrail monitoring.
diff --git a/files/plugins/check_contrail_analytics_alarms.py b/files/plugins/check_contrail_analytics_alarms.py
new file mode 100755
index 0000000..b76eb64
--- /dev/null
+++ b/files/plugins/check_contrail_analytics_alarms.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+
+import argparse
+import collections
+import datetime
+import ipaddress
+import os
+import os_client_config
+import requests
+import subprocess
+
+import nagios_plugin3
+
+
+def parse_contrail_alarms(data):
+    """Validate output data from Contrail Analytics Alarms section.
+
+    :param data: dict
+    :returns: str
+
+    The returned str shows a summary in the first line (as it will be displayed
+    in Nagios alerts). The rest of lines are sorted by timetamp (ts).
+    """
+    # If ack=False is found, crit_counter+=1; else, return WARNING
+    counter = crit_counter = 0
+    msgs_list = collections.defaultdict(lambda: [])
+    for node_type in data.keys():
+        # node_type: analytics-node, database-node, vrouter, ...
+        for item in data[node_type]:
+            # KVM, LXD or physical node hostname
+            hostname = item["name"]
+            # timestamp = item["value"]["UVEAlarms"]["__T"] / 1e6
+            for alarm in item["value"]["UVEAlarms"]["alarms"]:
+                ack = alarm["ack"]
+                alarm_info = {
+                    'hostname': hostname,
+                    'nagios_status': 'WARNING',
+                    'desc': alarm["description"],
+                    'sev': alarm["severity"],
+                    'ts': datetime.datetime.utcfromtimestamp(alarm["timestamp"] / 1e6),
+                    'type': alarm["type"],
+                }
+                counter += 1
+                if not ack or alarm["severity"] > 0:
+                    crit_counter += 1
+                    alarm_info["nagios_status"] = 'CRITICAL'
+
+                alarm_msg = ('{nagios_status}: {node_type}{{{hostname}, sev={sev},'
+                             ' ts[{ts}]}} {desc}'.format(
+                                 node_type=node_type, **alarm_info))
+                msgs_list[alarm["timestamp"]].append(alarm_msg)
+
+    if not msgs_list:
+        return 'OK: no alarms'
+
+    msg = 'CRITICAL: ' if crit_counter > 0 else 'WARNING: '
+    msg += 'total_alarms[{}], unacked_or_sev_gt_0[{}]\n{}'.format(
+        counter, crit_counter, '\n'.join(
+            '\n'.join(msgs_list[key]) for key in sorted(msgs_list)))
+    return msg
+
+
+def check_contrail_alarms(contrail_vip, token):
+    """Check the alarms in Contrail Analytics.
+
+    @param str vip: VIP of Contrail
+    @param str token: Token for the authentication
+    @returns: None
+    """
+    url = 'http://{}:8081/analytics/alarms'.format(contrail_vip)
+    headers = {'X-Auth-Token': token}
+    try:
+        r = requests.get(url=url, headers=headers)
+    except requests.exceptions.ConnectionError as error:
+        raise nagios_plugin3.CriticalError(
+            'CRITICAL: contrail analytics API error: {}'.format(error))
+
+    if r.code != 200:
+        raise nagios_plugin3.CriticalError(
+            'CRITICAL: contrail analytics API return code is {}'.format(r.code))
+
+    result = r.json()
+    msg = parse_contrail_alarms(result)
+
+    if msg.startswith('CRITICAL: '):
+        raise nagios_plugin3.CriticalError(msg)
+    elif msg.startswith('WARNING: '):
+        raise nagios_plugin3.WarnError(msg)
+    print('OK: no unacknowledged or sev>0 contrail analytics alarms')
+
+
+def load_os_envvars():
+    # grab environment vars
+    command = ['/bin/bash', '-c', "source {} && env".format(args.env)]
+    proc = subprocess.Popen(command, stdout=subprocess.PIPE)
+    for line in proc.stdout:
+        (key, _, value) = line.partition(b'=')
+        os.environ[key.decode('utf-8')] = value.rstrip().decode('utf-8')
+    proc.communicate()
+
+
+def validate_ipv4(ipv4_addr):
+    try:
+        ipaddress.IPv4Address(ipv4_addr)
+    except ipaddress.AddressValueError:
+        raise nagios_plugin3.UnknownError(
+            'UNKNOWN: invalid contrail IPv4 address {}'.format(ipv4_addr))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Check Contrail alarms')
+    parser.add_argument('--env', dest='env',
+                        default='/var/lib/nagios/nagios.novarc',
+                        help='Novarc file to use for this check')
+    parser.add_argument('--host', '-H', dest='host', nargs=1,
+                        help='Contrail Analytics Virtual IP')
+    args = parser.parse_args()
+
+    # Validate Contrail Analytics IP
+    contrail_analytics_vip = None
+    if isinstance(args.host, list):
+        contrail_analytics_vip = args.host[0]
+    nagios_plugin3.try_check(validate_ipv4, contrail_analytics_vip)
+
+    # Retrieve token from Keystone
+    load_os_envvars()
+    keystone_client = os_client_config.session_client('identity', cloud='envvars')
+    token = keystone_client.get_token()
+
+    nagios_plugin3.try_check(check_contrail_alarms, contrail_analytics_vip, token)
diff --git a/lib/lib_openstack_service_checks.py b/lib/lib_openstack_service_checks.py
index fd12f51..8d654ef 100644
--- a/lib/lib_openstack_service_checks.py
+++ b/lib/lib_openstack_service_checks.py
@@ -41,6 +41,10 @@ class OSCHelper():
         return '/var/lib/nagios/nagios.novarc'
 
     @property
+    def contrail_analytics_vip(self):
+        return self.charm_config['contrail_analytics_vip']
+
+    @property
     def plugins_dir(self):
         return '/usr/local/lib/nagios/plugins/'
 
@@ -175,6 +179,17 @@ class OSCHelper():
                                               'check_neutron_agents.sh'),
                        )
 
+        if self.contrail_analytics_vip:
+            contrail_check_command = '{} --host {}'.format(
+                os.path.join(self.plugins_dir, 'check_contrail_analytics_alarms.py'),
+                self.contrail_analytics_vip)
+            nrpe.add_check(shortname='contrail_analytics_alarms',
+                           description='Check Contrail Analytics alarms',
+                           check_cmd=contrail_check_command,
+                           )
+        else:
+            nrpe.remove_check(shortname='contrail_analytics_alarms')
+
         if len(self.check_dns):
             nrpe.add_check(shortname='dns_multi',
                            description='Check DNS names are resolvable',
diff --git a/tests/functional/test_deploy.py b/tests/functional/test_deploy.py
index 30ca92d..5ef53d8 100644
--- a/tests/functional/test_deploy.py
+++ b/tests/functional/test_deploy.py
@@ -188,3 +188,34 @@ async def test_openstackservicechecks_enable_rally(deploy_app, model, file_stat)
     for filename in filenames:
         test_stat = await file_stat(filename, unit)
         assert test_stat['size'] > 0
+
+
+async def test_openstackservicechecks_enable_contrail_analytics_vip(deploy_app, model, file_stat):
+    unit = [unit for unit in model.units.values() if unit.entity_id.startswith(deploy_app.name)]
+    if len(unit) != 1:
+        assert False
+
+    unit = unit[0]
+    filename = '/etc/nagios/nrpe.d/check_contrail_analytics_alarms.cfg'
+
+    # disable rally nrpe check if it was enabled (ie. from a previous run of functests)
+    config = await deploy_app.get_config()
+    if config['contrail_analytics_vip']['value']:
+        await deploy_app.set_config({'contrail_analytics_vip': ''})
+        # Wait until nrpe check is set
+        await model.block_until(lambda: deploy_app.status == 'active' and unit.agent_status == 'idle',
+                                timeout=600)
+
+    # Check BEFORE enabling contrail_analytics_vip
+    # raises exception because filename does not exist
+    with pytest.raises(json.decoder.JSONDecodeError):
+        await file_stat(filename, unit)
+
+    await deploy_app.set_config({'contrail_analytics_vip': '127.0.0.1'})
+    # Wait until nrpe check is set
+    await model.block_until(lambda: deploy_app.status == 'active' and unit.agent_status == 'idle',
+                            timeout=600)
+
+    # Check AFTER enabling contrail_analytics_vip
+    test_stat = await file_stat(filename, unit)
+    assert test_stat['size'] > 0