Merge ~aieri/charm-prometheus-blackbox-exporter:bb-sub into ~prometheus-charmers/charm-prometheus-blackbox-exporter:master

[Andrea Ieri]

Blackbox exporter rewritten as subordinate

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Unable to determine commit message from repository - please click "Set commit message" and enter the commit message manually.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Unable to determine commit message from repository - please click "Set commit message" and enter the commit message manually.

7927384... by Andrea Ieri on 2019-09-18

Add the juju-info interface explicitly

413057f... by Andrea Ieri on 2019-09-19

Publish the external IP in case we're running cross-model relations

64d1422... by Andrea Ieri on 2019-09-25

Filter out uninteresting IPs and interfaces

881b419... by Andrea Ieri on 2019-09-26

Lookup source ips for all probes and provide the address over the relation

a458c6a... by Andrea Ieri on 2019-09-26

Noop readability refactor

[Alvaro Uria (aluria)]

A new charm needs to be created (ie. prometheus-blackbox-peer-exporter) for this MP. The current charm is non-subordinate and used by IS, while the current fix makes the charm a subordinate.

I will talk to JL to create that new charm and create MPs against it.

review: Disapprove

Unmerged commits

a458c6a... by Andrea Ieri on 2019-09-26

Noop readability refactor

881b419... by Andrea Ieri on 2019-09-26

Lookup source ips for all probes and provide the address over the relation

64d1422... by Andrea Ieri on 2019-09-25

Filter out uninteresting IPs and interfaces

413057f... by Andrea Ieri on 2019-09-19

Publish the external IP in case we're running cross-model relations

7927384... by Andrea Ieri on 2019-09-18

Add the juju-info interface explicitly

f631577... by Andrea Ieri on 2019-09-11

Fix subordinate endpoint name

af6e21c... by Andrea Ieri on 2019-09-11

Cosmetic changes to layer/metadata. Make charm proof a little happier.

4bd9ed2... by Andrea Ieri on 2019-09-11

Add a wheelhouse.txt to cover python dependencies

60d1957... by Diko Parvanov on 2019-09-09

Added icmp checks and fixed hooks/peer relations

347f56e... by Diko Parvanov on 2019-09-08

Added multiple networks checks + MTU config option

diff --git a/config.yaml b/config.yaml
index d1dc24d..fc04ec0 100644
--- a/config.yaml
+++ b/config.yaml
@@ -16,6 +16,13 @@ options:
       icmp:
         prober: icmp
         timeout: 10s
+        icmp:
+          preferred_ip_protocol: "ip4"
+          payload_size: 1472
     type: string
     description: |
         Blackbox exporter configuratin in raw YAML format
+  scrape-interval:
+    type: string
+    default: "60s"
+    description: Set the blackbox exporter scrape jobs custom interval.
diff --git a/layer.yaml b/layer.yaml
index ddd95fc..6036e6d 100644
--- a/layer.yaml
+++ b/layer.yaml
@@ -1,6 +1,11 @@
-includes: ['layer:basic', 'interface:http', 'layer:snap']
+includes:
+  - 'layer:basic'
+  - "interface:juju-info"
+  - 'interface:http'
+  - 'interface:peer-discovery'
 repo: 'https://git.launchpad.net/prometheus-blackbox-exporter-charm'
-ignore: ['.*.swp' ]
+ignore:
+  - '.*.swp'
 options:
   basic:
     use_venv: true
diff --git a/metadata.yaml b/metadata.yaml
index 4f9228b..89d3c6e 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -1,4 +1,5 @@
 name: prometheus-blackbox-exporter
+display-name: Prometheus Blackbox Exporter
 summary: Blackbox exporter for Prometheus
 maintainer: Jacek Nykis <[e-mail]>
 description: |
@@ -7,9 +8,15 @@ description: |
 tags:
   - monitoring
 series:
-  - xenial
   - bionic
-subordinate: false
+subordinate: true
 provides:
   blackbox-exporter:
     interface: http
+requires:
+  general-info:
+    interface: juju-info
+    scope: container
+peers:
+  blackbox-peer:
+    interface: peer-discovery
diff --git a/reactive/prometheus-blackbox-exporter.py b/reactive/prometheus-blackbox-exporter.py
index a5d792a..7d4e877 100644
--- a/reactive/prometheus-blackbox-exporter.py
+++ b/reactive/prometheus-blackbox-exporter.py
@@ -1,19 +1,33 @@
+import ast
+import re
+import subprocess
 import yaml
+import sys
 
 from charmhelpers.core import host, hookenv
 from charmhelpers.core.templating import render
 from charms.reactive import (
     when, when_not, set_state, remove_state
 )
+from ipaddress import (
+    IPv4Interface, IPv4Address
+)
+from pyroute2 import IPRoute
+from netifaces import interfaces, ifaddresses, AF_INET
 from charms.reactive.helpers import any_file_changed, data_changed
-from charms.layer import snap
+from charms.reactive import hook
+
+from charmhelpers.fetch import apt_install
 
+hooks = hookenv.Hooks()
 
-SNAP_NAME = 'prometheus-blackbox-exporter'
-SVC_NAME = 'snap.prometheus-blackbox-exporter.daemon'
+APT_PKG_NAME = 'prometheus-blackbox-exporter'
+SVC_NAME = 'prometheus-blackbox-exporter'
+EXECUTABLE = '/usr/bin/prometheus-blackbox-exporter'
 PORT_DEF = 9115
 BLACKBOX_EXPORTER_YML_TMPL = 'blackbox.yaml.j2'
-CONF_FILE_PATH = '/var/snap/prometheus-blackbox-exporter/current/blackbox.yml'
+CONF_FILE_PATH = '/etc/prometheus/blackbox.yml'
+IFACE_BLACKLIST_PATTERN = re.compile('^(lo|virbr|docker|lxdbr|vhost|tun|tap)')
 
 
 def templates_changed(tmpl_list):
@@ -24,8 +38,9 @@ def templates_changed(tmpl_list):
 def install_packages():
     hookenv.status_set('maintenance', 'Installing software')
     config = hookenv.config()
-    channel = config.get('snap_channel', 'stable')
-    snap.install(SNAP_NAME, channel=channel, force_dangerous=False)
+    apt_install(APT_PKG_NAME, fatal=True)
+    cmd = ["sudo", "setcap", "cap_net_raw+ep", EXECUTABLE]
+    subprocess.check_output(cmd)
     set_state('blackbox-exporter.installed')
     set_state('blackbox-exporter.do-check-reconfig')
 
@@ -86,9 +101,150 @@ def restart_blackbox_exporter():
     set_state('blackbox-exporter.started')
     remove_state('blackbox-exporter.do-restart')
 
-
 # Relations
-@when('blackbox-exporter.started')
-@when('blackbox-exporter.available') # Relation name is "blackbox-exporter"
-def configure_blackbox_exporter_relation(target):
-    target.configure(PORT_DEF)
+@hook('blackbox-peer-relation-{joined,departed}')
+def configure_blackbox_exporter_relation(peers):
+    hookenv.log('Running blackbox exporter relation.')
+    hookenv.status_set('maintenance', 'Configuring blackbox peer relations.')
+    config = hookenv.config()
+
+    icmp_targets = []
+    tcp_targets = []
+    networks = []
+    for rid in hookenv.relation_ids('blackbox-peer'):
+        for unit in hookenv.related_units(rid):
+            unit_ports = hookenv.relation_get('unit-ports', rid=rid, unit=unit)
+            principal_unit = hookenv.relation_get('principal-unit', rid=rid, unit=unit)
+            unit_networks = hookenv.relation_get('unit-networks', rid=rid, unit=unit)
+            if unit_networks is not None:
+                unit_networks = ast.literal_eval(unit_networks)
+                for unit_network in unit_networks:
+                    # Chcek if same network exists on this unit
+                    if unit_network['net'] in [net['net'] for net in get_unit_networks()]:
+                        networks.append(unit_network['net'])
+                        probe_dst_ip = unit_network['ip']
+                        icmp_targets.append({
+                            'network': unit_network['net'],
+                            'interface': unit_network['iface'],
+                            'ip-address': probe_dst_ip,
+                            'principal-unit': principal_unit,
+                            'module': 'icmp',
+                            'source-ip': _get_source_ip(probe_dst_ip)
+                            })
+
+            if unit_ports is not None:
+                unit_ports = ast.literal_eval(unit_ports)
+                for port in unit_ports:
+                    tcp_targets.append({
+                        'ip-address': unit_network['ip'],
+                        'port': port,
+                        'principal-unit': principal_unit,
+                        'module': 'tcp_connect',
+                        })
+
+    relation_settings = {}
+    relation_settings['icmp_targets'] = icmp_targets
+    relation_settings['tcp_targets'] = tcp_targets
+    relation_settings['networks'] = networks
+    relation_settings['ip_address'] = hookenv.unit_get('private-address')
+    relation_settings['port'] = PORT_DEF
+    relation_settings['job_name'] = hookenv.principal_unit()
+    relation_settings['scrape_interval'] = config.get('scrape-interval')
+
+
+    for rel_id in hookenv.relation_ids('blackbox-exporter'):
+        relation_settings['ip_address'] = \
+            hookenv.ingress_address(rid=rel_id, unit=hookenv.local_unit())
+        hookenv.relation_set(relation_id=rel_id, relation_settings=relation_settings)
+
+    hookenv.status_set('active', 'Ready')
+
+
+def _get_source_ip(destination):
+    """
+    Get the source ip of a connection towards destination without having to run
+    ip r g via subprocess
+
+    Disclaimer: source ip configuration for the blackbox exporter is done in
+    the module definition. To avoid having to create a module for every local
+    address we are simply letting the exporter use whatever source IP the
+    kernel deems appropriate. Be aware that this function is doing a route
+    lookup, and it is not - strictly speaking - returning the source IP of the
+    packet the blackbox exporter will generate. The two addresses should always
+    be the same, but YMMV.
+    """
+    with IPRoute() as ipr:
+        routes = ipr.route('get', dst=destination)
+        return routes[0].get_attr('RTA_PREFSRC')
+
+
+def _is_valid_ip(address):
+    """
+    Filter out "uninteresting" addresses
+    """
+    ip = IPv4Address(address.get('addr'))
+    return not (ip.is_multicast or
+                ip.is_reserved or
+                ip.is_link_local or
+                ip.is_loopback)
+
+
+def _is_valid_iface(iface):
+    """
+    Ignore interfaces used by Docker, KVM, Contrail, etc
+    """
+    if IFACE_BLACKLIST_PATTERN.search(iface):
+        return False
+    else:
+        return True
+
+
+def get_unit_networks():
+    networks = []
+    for iface in filter(_is_valid_iface, interfaces()):
+        ip_addresses = ifaddresses(iface)
+        for ip_address in filter(_is_valid_ip, ip_addresses.get(AF_INET, [])):
+            ip_v4 = IPv4Interface(
+                "{}/{}".format(ip_address['addr'], ip_address['netmask'])
+            )
+            networks.append(
+                {"iface": iface,
+                 "ip": str(ip_v4.ip),
+                 "net": str(ip_v4.network)}
+            )
+    return networks
+
+def get_principal_unit_open_ports():
+    cmd = "lsof -P -iTCP -sTCP:LISTEN".split()
+    result = subprocess.check_output(cmd)
+    result = result.decode(sys.stdout.encoding)
+
+    ports = []
+    for r in result.split('\n'):
+        for p in r.split():
+            if '*:' in p:
+                ports.append(p.split(':')[1])
+    ports = [p for p in set(ports)]
+
+    return ports
+
+@hook('blackbox-peer-relation-{joined,departed}')
+def blackbox_peer_departed(peers):
+    hookenv.log('Blackbox peer unit joined/departed.')
+    set_state('blackbox-exporter.redo-peer-relation')
+
+@when('blackbox-exporter.redo-peer-relation')
+def setup_blackbox_peer_relation(peers):
+    # Set blackbox-peer relations
+    hookenv.log('Running blackbox peer relations.')
+    hookenv.status_set('maintenance', 'Configuring blackbox peer relations.')
+    for rid in hookenv.relation_ids('blackbox-peer'):
+        relation_settings = hookenv.relation_get(rid=rid, unit=hookenv.local_unit())
+        relation_settings['principal-unit'] = hookenv.principal_unit()
+        relation_settings['private-address'] = hookenv.unit_get('private-address')
+        relation_settings['unit-networks'] = get_unit_networks()
+        relation_settings['unit-ports'] = get_principal_unit_open_ports()
+        hookenv.relation_set(relation_id=rid, relation_settings=relation_settings)
+
+    hookenv.status_set('active', 'Ready')
+    remove_state('blackbox-exporter.redo-peer-relation')
diff --git a/wheelhouse.txt b/wheelhouse.txt
 new file mode 100644
index 0000000..ea2fb5e
--- /dev/null
+++ b/wheelhouse.txt
@@ -0,0 +1,2 @@
+netifaces
+pyroute2