Merge ~hloeung/charm-graylog:master into ~graylog-charmers/charm-graylog:master

[Haw Loeung]

Add log inputs from charm config

This adds new 'log_inputs' charm config option to define a set of log inputs. These inputs are then set up via the Graylog API and ports opened (juju open-port) so that a juju expose sets up the correct secgroup rules.

While this works, I'm not proud of it. I have a task to split the API parts out to it's own library.

[Tim Kuhlman (timkuhlman)]

Nice new functionality!

I think the implementation can be improved with some unit tests and by pulling all of the Graylog API code into a library, ideally one that can be shared between the reactive code and the amulet tests.

Also this is pretty significant new functionality, the Amulet tests should be extended to cover setting and verifying log_inputs.

review: Needs Fixing

0846e8f... by Haw Loeung on 2017-08-17

Add log inputs from charm config

This adds new 'log_inputs' charm config option to define a set of log
inputs. These inputs are then set up via the Graylog API and ports
opened (juju open-port) so that a juju expose sets up the correct
secgroup rules.

[Tim Kuhlman (timkuhlman)]

I took a brief look at the updates with some comments below. The changes look good but I would still like to see unit and/or amulet tests and at some point a class for the Graylog API.

962a4a4... by Haw Loeung on 2017-08-23

Add unit tests for input type map and input exists

Unmerged commits

962a4a4... by Haw Loeung on 2017-08-23

Add unit tests for input type map and input exists

0846e8f... by Haw Loeung on 2017-08-17

Add log inputs from charm config

This adds new 'log_inputs' charm config option to define a set of log
inputs. These inputs are then set up via the Graylog API and ports
opened (juju open-port) so that a juju expose sets up the correct
secgroup rules.

diff --git a/TODO b/TODO
index e0c4f88..2272c2a 100644
--- a/TODO
+++ b/TODO
@@ -1,8 +1,7 @@
 Before Production Use:
 - Clustering of multiple graylog instances.
-- Support for nrpe relation, including service specific nagios check.
 - Support for removing relations.
-- Configuration of Inputs, specifically beats setup, most likely via the REST API.
+- Support for logstash relation.
   - When charm-based configuration of beats is in place, support the logstash relation so that
     filebeats can use that relation to setup the proper output when in the same Juju model.
 
diff --git a/config.yaml b/config.yaml
index 8db8a8c..9614533 100644
--- a/config.yaml
+++ b/config.yaml
@@ -47,6 +47,17 @@ options:
     default: 6
     description: |
       Maximum number of indices to keep before deleting the oldest ones
+  log_inputs:
+    type: string
+    default: |
+      - name: Beats Input
+        type: Beats
+        bind_address: 0.0.0.0
+        bind_port: 5044
+    description: |
+      YAML-formatted list of log inputs. First input gets passed
+      through relations. Any input not defined here will be removed unless it
+      is prefixed with "Custom" in the title.
   nagios_context:
     default: "juju"
     type: string
diff --git a/metadata.yaml b/metadata.yaml
index b5f3096..8277908 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -1,6 +1,6 @@
 name: graylog
 summary: Graylog log management system
-maintainer: Tim Kuhlman <[e-mail]>
+maintainer: Graylog Charmers <[e-mail]>
 description: >
   Installs the Graylog log management system. Connections to elasticsearch and
   mongodb are required for a fully functioning system. https://www.graylog.org/
diff --git a/reactive/graylog.py b/reactive/graylog.py
index 016f061..a93b5e8 100644
--- a/reactive/graylog.py
+++ b/reactive/graylog.py
@@ -4,6 +4,7 @@ import os
 import re
 import requests
 import time
+import yaml
 from urllib.parse import urlparse, urljoin
 
 from charms.reactive import hook, when, when_not, remove_state, set_state
@@ -30,8 +31,6 @@ def update_config():
 
 
 def api_request(url, auth, method='GET', data=None):
-    if method not in ('GET', 'POST', 'PUT'):
-        return
     tries = 0
     while tries < 3:
         tries += 1
@@ -40,6 +39,8 @@ def api_request(url, auth, method='GET', data=None):
                                         headers={'Accept': 'application/json',
                                                  'Content-Type': 'application/json'})
             if resp.ok:
+                if method == 'DELETE':
+                    return True
                 return json.loads(resp.content.decode('utf-8'))
         except Exception:
             pass
@@ -60,11 +61,6 @@ def api_retrieve_token(api_base_url, auth, name='graylog-charm'):
     return api_retrieve_token(api_base_url, auth, name=name)
 
 
-def api_retrieve_index_sets(api_base_url, auth):
-    url = urljoin(api_base_url, 'system/indices/index_sets')
-    return api_request(url, auth=auth)['index_sets']
-
-
 rotation_strategies = {
     'time': {
         'class': 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy',
@@ -111,14 +107,29 @@ def configure_graylog():
 @when('elasticsearch.available')
 @when_not('graylog.needs_restart')
 @when_not('graylog_api.configured')
-def configure_graylog_via_api(*discard):
-    conf = hookenv.config()
+def configure_graylog_api(*discard):
     db = unitdata.kv()
     admin_password = db.get('admin_password')
     api_token = api_retrieve_token(API_URL, auth=('admin', admin_password))
     if not api_token:
         return
-    index_sets = api_retrieve_index_sets(API_URL, auth=(api_token, 'token'))
+    if db.get('api_token') != api_token:
+        db.set('api_token', api_token)
+    remove_state('graylog_index_sets.configured')
+    remove_state('graylog_inputs.configured')
+    set_state('graylog_api.configured')
+
+
+@when('graylog_api.configured')
+@when_not('graylog_index_sets.configured')
+def configure_index_sets(*discard):
+    conf = hookenv.config()
+    db = unitdata.kv()
+    api_token = db.get('api_token')
+    if not api_token:
+        return
+    url = urljoin(API_URL, 'system/indices/index_sets')
+    index_sets = api_request(url, auth=(api_token, 'token'))['index_sets']
     if not index_sets:
         return
     for iset in index_sets:
@@ -141,7 +152,154 @@ def configure_graylog_via_api(*discard):
                            data=json.dumps(iset, indent=True)):
             return
 
-    set_state('graylog_api.configured')
+    set_state('graylog_index_sets.configured')
+
+
+def _map_input_type(input_types, type):
+    """
+    Map specified int types to Gray's input type objects. Examples:
+
+      "Beats" -> org.graylog.plugins.beats.BeatsInput
+      "Raw TCP" -> org.graylog2.inputs.raw.tcp.RawTCPInput
+      "Syslog UDP" -> org.graylog2.inputs.syslog.udp.SyslogUDPInput
+    """
+    for k, v in input_types.items():
+        if v.lower().find(type.lower()) != -1:
+            return k
+        # For raw input types, map something like "Raw TCP" and "Raw UDP"
+        # instead of requiring the exact name, "Raw/Plaintext TCP".
+        if k.lower().find('.'.join(type.split()).lower()) != -1:
+            return k
+    return None
+
+
+def _check_input_exists(current_inputs, new):
+    """
+    Given the current list of configured Graylog inputs, check if new input
+    exists and return an ID as well as "changed" flag to indicate that the
+    new input provided has changed (input type, bind address, and port).
+    """
+    changed = True
+    cur = None
+    for input in current_inputs:
+        if input['title'] == new['title']:
+            cur = input
+            break
+    if not cur:
+        return (True, None)
+    current = [cur['type'], cur['attributes']['bind_address'], str(cur['attributes']['port'])]
+    new = [new['type'], new['configuration']['bind_address'], str(new['configuration']['port'])]
+    if sorted(new) == sorted(current):
+        changed = False
+    return (changed, cur['id'])
+
+
+def _close_uneeded_ports(current_inputs, new):
+    for cur in current_inputs:
+        title = cur['title']
+        if title.lower().startswith('custom'):
+            continue
+        type = cur['type']
+        if 'udp' in type.lower():
+            proto = 'UDP'
+        else:
+            proto = 'TCP'
+        port = cur['attributes']['port']
+        if '{}/{}'.format(port, proto) not in new:
+            hookenv.close_port(port, proto)
+
+
+def _remove_old_inputs(inputs, new_inputs):
+    """
+    Removes any old/previously defined inputs, or any unknown. We'll keep those
+    prefixed with 'Custom' to allow configuring additional inputs via the web
+    UI.
+    """
+    db = unitdata.kv()
+    api_token = db.get('api_token')
+    success = True
+    for cur in inputs:
+        title = cur['title']
+        if title.lower().startswith('custom'):
+            continue
+        if title in new_inputs:
+            continue
+        url = urljoin(urljoin(API_URL, 'system/inputs/'), cur['id'])
+        if api_request(url, auth=(api_token, 'token'), method='DELETE'):
+            hookenv.log('Removed old input: {} ({})'.format(cur['title'], cur['id']))
+        else:
+            success = False
+    return success
+
+
+@when('graylog_api.configured')
+@when_not('graylog_inputs.configured')
+def configure_inputs(*discard):
+    """
+    Configure log inputs in Graylog via the API.
+    """
+    conf = hookenv.config()
+    db = unitdata.kv()
+    api_token = db.get('api_token')
+    if not api_token:
+        return
+    url = urljoin(API_URL, 'system/inputs')
+    inputs = api_request(url, auth=(api_token, 'token'))['inputs']
+    url = urljoin(API_URL, 'system/inputs/types')
+    input_types = api_request(url, auth=(api_token, 'token'))['types']
+    new_opened_ports = []
+    new_inputs = []
+    for new in yaml.safe_load(conf['log_inputs']) or {}:
+        type = _map_input_type(input_types, new['type'])
+        if not type:
+            hookenv.log('Input type "{}" not supported'.format(new['type']))
+            continue
+        d = {
+            'title': new['name'],
+            'type': type,
+            'global': "true",
+            'configuration': {
+                'bind_address': new['bind_address'],
+                'port': new['bind_port'],
+            }
+        }
+        if 'udp' in type.lower():
+            proto = 'UDP'
+        else:
+            proto = 'TCP'
+            d['configuration']['tcp_keepalive'] = 'true'
+
+        hookenv.open_port(new['bind_port'], proto)
+        new_opened_ports.append('{}/{}'.format(new['bind_port'], proto))
+
+        (changed, input_id) = _check_input_exists(inputs, d)
+        if not changed:
+            new_inputs.append(new['name'])
+            continue
+        if input_id:
+            change_text = 'Updated existing input'
+            method = 'PUT'
+            url = urljoin(urljoin(API_URL, 'system/inputs/'), input_id)
+        else:
+            change_text = 'Adding new input'
+            method = 'POST'
+            url = urljoin(API_URL, 'system/inputs/')
+        ret = api_request(url, auth=(api_token, 'token'), method=method,
+                          data=json.dumps(d, indent=True))
+        if not ret:
+            # Can't add input, let's try again later
+            return
+        hookenv.log('{}: {} ({})'.format(change_text, new['name'], ret['id']))
+        new_inputs.append(new['name'])
+
+    # Now remove inputs that should no longer be there as well as close ports
+    _close_uneeded_ports(inputs, new_opened_ports)
+    url = urljoin(API_URL, 'system/inputs')
+    refreshed_inputs = api_request(url, auth=(api_token, 'token'))['inputs']
+    if not _remove_old_inputs(refreshed_inputs, new_inputs):
+        # Can't delete input, let's try again later
+        return
+    set_state('graylog_inputs.configured')
 
 
 @when('graylog.configured')
diff --git a/unit_tests/graylog.py b/unit_tests/graylog.py
index 0ee4134..2b16c78 100644
--- a/unit_tests/graylog.py
+++ b/unit_tests/graylog.py
@@ -5,7 +5,7 @@ import unittest
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 
-from reactive.graylog import set_conf
+from reactive.graylog import set_conf, _map_input_type, _check_input_exists
 
 
 initial_conf = u"""#key1 = value1
@@ -38,6 +38,25 @@ key2 = value6
 ### key3   =  value 4
 key4 = value5"""
 
+log_input_types = {
+    "org.graylog.plugins.beats.BeatsInput": "Beats",
+    "org.graylog2.inputs.raw.udp.RawUDPInput": "Raw/Plaintext UDP",
+}
+
+log_inputs = [{
+    "title": "Beats Input",
+    "global": "true",
+    "name": "Beats",
+    "created_at": "2017-08-17T10:19:10.634Z",
+    "type": "org.graylog.plugins.beats.BeatsInput",
+    "creator_user_id": "admin",
+    "attributes": {
+        "bind_address": "0.0.0.0",
+        "port": 11044
+    },
+    "id": "5994f3aab0d37f23c38adbda"
+}]
+
 
 class TestSetConf(unittest.TestCase):
 
@@ -81,3 +100,48 @@ class TestSetConf(unittest.TestCase):
 
     def test_changed(self):
         self.assertTrue(set_conf('key2', 'some-new-value', self.conf_file))
+
+    def test_input_type_exists(self):
+        self.assertTrue(_map_input_type(log_input_types, 'beats'))
+
+    def test_input_type_raw(self):
+        self.assertTrue(_map_input_type(log_input_types, 'raw udp'))
+
+    def test_input_type_invalid(self):
+        self.assertFalse(_map_input_type(log_input_types, 'some-weird-type'))
+
+    def test_input_exists(self):
+        new_input = {
+            "title": "Beats Input",
+            "type": "org.graylog.plugins.beats.BeatsInput",
+            "configuration": {
+                "bind_address": "0.0.0.0",
+                "port": 11044
+            }
+        }
+        (changed, input_id) = _check_input_exists(log_inputs, new_input)
+        self.assertTrue(input_id == "5994f3aab0d37f23c38adbda" and changed is False)
+
+    def test_input_changed(self):
+        new_input = {
+            "title": "Beats Input",
+            "type": "org.graylog.plugins.beats.BeatsInput",
+            "configuration": {
+                "bind_address": "0.0.0.0",
+                "port": 12222
+            }
+        }
+        (changed, input_id) = _check_input_exists(log_inputs, new_input)
+        self.assertTrue(changed is True)
+
+    def test_input_new(self):
+        new_input = {
+            "title": "Beats Input 2",
+            "type": "org.graylog.plugins.beats.BeatsInput",
+            "configuration": {
+                "bind_address": "0.0.0.0",
+                "port": 11033
+            }
+        }
+        (changed, input_id) = _check_input_exists(log_inputs, new_input)
+        self.assertTrue(input_id is None)