Merge lp:~cprov/charms/trusty/logstash/heap_size into lp:~tanuki/charms/trusty/logstash/trunk

[Celso Providelo]

Allowing changing the logstash maximum heap size as a configuration.

Allowing changing the logstash maximum heap size as a configuration.

Defaults to 1000m instead of the application 500m default value and allow adjustments.

[lp:~cprov/charms/trusty/logstash/heap_size]

59. By Celso Providelo on 2016-11-23

Adding heap_size config option.

[Guillermo Gonzalez (verterok)]

Celso,

If this is to apply it to the OLS ELK, it's using a a different charm:

lp:~canonical-is-sa/charms/trusty/logstash/logstash2;revno=61,overwrite=true

the spec in use (for both staging and production) is: is/mojo-is-logging-kibana/ @ lp:canonical-mojo-specs.

review: Needs Information

[Michael Nelson (michael.nelson)]

LGTM. Note that in the ES charm, we're explicitly giving unlimited memlock. See the task at:

http://bazaar.launchpad.net/~onlineservices-charmers/charms/trusty/elasticsearch/elasticsearch2/view/head:/tasks/install-elasticsearch.yml#L49

which installs this template:

http://bazaar.launchpad.net/~onlineservices-charmers/charms/trusty/elasticsearch/elasticsearch2/view/head:/templates/limits.conf

review: Approve

[lp:~cprov/charms/trusty/logstash/heap_size]

60. By Celso Providelo on 2016-11-23

merge right branch (lp:~canonical-is-sa/charms/trusty/logstash/logstash2).

61. By Celso Providelo on 2016-11-23

Special (no-) limits configuration for logstash, so it can eat as much memory it is configured too.

Unmerged revisions

61. By Celso Providelo on 2016-11-23

Special (no-) limits configuration for logstash, so it can eat as much memory it is configured too.

60. By Celso Providelo on 2016-11-23

merge right branch (lp:~canonical-is-sa/charms/trusty/logstash/logstash2).

59. By Celso Providelo on 2016-11-23

Adding heap_size config option.

=== modified file 'README.md'
--- README.md	2015-06-30 16:42:53 +0000
+++ README.md	2016-11-23 22:25:27 +0000
@@ -54,27 +54,6 @@
     juju add-relation logstash-agent logstash-indexer:input
 
 
-example 4  - python-logstash + Indexer + 2x ElasticSearch + Kibana
-==================================================================
-
-    juju deploy cs:trusty/elasticsearch
-    juju add-unit elasticsearch
-    juju deploy cs:trusty/logstash-indexer
-    juju add-relation elasticsearch:cluster logstash-indexer
-    juju deploy cs:trusty/kibana
-    juju add-relation elasticsearch:rest kibana
-    juju expose kibana
-
-    juju expose logstash-indexer
-    $ python3
-    >>> import logging, logstash
-    >>> logger = logging.getLogger()
-    >>> logger.addHandler(
-    ...     logstash.LogstashHandler('<ip-of-indexer>', port=5959, version=1)) 
-    >>> logger.info('Hello Logstash!!!')
-
-http://ip-of-kibana
-
 ### Caveats
 
 The charm will fetch the logstash complete archive every time. 

=== modified file 'config.yaml'
--- config.yaml	2015-09-29 19:30:00 +0000
+++ config.yaml	2016-11-23 22:25:27 +0000
@@ -1,11 +1,11 @@
 options:
   logstash-source:
     type: string
-    default: "https://download.elasticsearch.org/logstash/logstash/logstash-1.4.2.tar.gz"
+    default: "https://download.elastic.co/logstash/logstash/logstash-2.0.0.tar.gz"
     description: The logstash binary file to install to this charm.
   logstash-sum:
     type: string
-    default: "d59ef579c7614c5df9bd69cfdce20ed371f728ff"
+    default: "f0961520dd9590d3b600c877be66f79f94a05f80"
     description: The checksum value for the logstash file.
   extra-packages:
     type: string
@@ -22,7 +22,29 @@
   extra-config:
     type: string
     default: ''
-    description: "Base64-encoded custom configuration content."
+    description: |
+      Base64-encoded custom configuration content. If you have a directory of
+      logstash filters, you can set this with:
+          juju set logstash extra-config="$(cat filters/* | base64)"
+  extra-patterns:
+    type: string
+    default: ''
+    description: |
+      Base64-encoded custom grok patterns for your logstash filters. If you
+      have a directory of patterns, you can set this with:
+          juju set logstash extra-patterns="$(cat patterns/* | base64)"
+      These will be placed into the /opt/logstash/patterns directory and so can be
+      referenced in a grok filter with:
+          grok {
+            patterns_dir => "./patterns"
+            match => { ... }
+          }
+      More info at https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html#_custom_patterns
+  heap_size:
+    default: "1000m"
+    type: string
+    description: |
+      Maximum Java heap size for the logstash process.
   nagios_context:
     default: "juju"
     type: string
@@ -40,7 +62,7 @@
       A comma-separated list of nagios servicegroups.
       If left empty, the nagios_context will be used as the servicegroup
   nagios_check_procs_params:
-    default: "-a /opt/logstash/lib/logstash/runner.rb -c 1:1"
+    default: "-a logstash/runner.rb -c 1:1"
     type: string
     description: The parameters to pass to the nrpe plugin check_procs.
   nagios_check_tcp_params:
@@ -51,3 +73,11 @@
     default: "-D 30,14 -H 127.0.0.1 -p 5043"
     type: string
     description: The parameters to pass to the nrpe plugin "check_tcp --ssl" to check certificate expiration date.
+  nagios_check_e2e_params:
+    default: "-w 30 -c 60"
+    type: string
+    description: The parameters to pass to the nrpe plugin "check_lumberjack_e2e.py" to do a E2E check.
+  nagios_check_e2e_cron:
+    default: ""
+    type: string 
+    description: cron interval

=== added directory 'files/nrpe'
=== added file 'files/nrpe/check_e2e_output.py'
--- files/nrpe/check_e2e_output.py	1970-01-01 00:00:00 +0000
+++ files/nrpe/check_e2e_output.py	2016-11-23 22:25:27 +0000
@@ -0,0 +1,59 @@
+#!/usr/bin/env python2
+"""Script to check the output of check_lumberjack_e2e.py"""
+
+# Copyright (C) 2016  Guillermo Gonzalez <[e-mail]>
+
+from __future__ import unicode_literals
+from __future__ import print_function
+
+import os
+import sys
+import time
+import traceback
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', dest="e2e_file", required=True)
+    parser.add_argument('-w', dest="warn", type=int, default=450,
+                        help="file age warning threshold.")
+    parser.add_argument('-c', dest="critical", type=int, default=600,
+                        help="file age critical threshold")
+    args = parser.parse_args()
+
+    if os.path.exists(args.e2e_file):
+        # check file age
+        file_age = time.time() - os.path.getmtime(args.e2e_file)
+        if file_age > args.critical:
+            print("ERROR: e2e output file is too old (%s)" % (file_age))
+            return 2
+        elif file_age < args.critical and file_age > args.warn:
+            print("WARNING: e2e output file is old (%s)" % (file_age))
+            return 1
+        # the file is recent enough
+        else:
+            with open(args.e2e_file, 'r') as fd:
+                lines = fd.readlines()
+                output = lines[-1]
+            if output.startswith("OK"):
+                print(output)
+                return 0
+            elif output.startswith("WARNING"):
+                print(output)
+                return 1
+            elif output.startswith("ERROR"):
+                print(output)
+                return 2
+    else:
+        print("ERROR: e2e output file (%s) is missing" % (args.e2e_file,))
+        return 2
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception, e:
+        print("ERROR: Unhandled error: %s" % (str(e),))
+        traceback.print_exc()
+        sys.exit(3)

=== added file 'files/nrpe/check_lumberjack_e2e.py'
--- files/nrpe/check_lumberjack_e2e.py	1970-01-01 00:00:00 +0000
+++ files/nrpe/check_lumberjack_e2e.py	2016-11-23 22:25:27 +0000
@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""
+This is very simple lumbjerjack client to run a e2e check against logstash+elasticsearch.
+
+All the lumberjack protocol code was adapted from Matt Johnson: https://pypi.python.org/pypi/stashward/0.1.4
+"""
+
+import traceback
+import datetime
+import struct
+import errno
+import socket
+import sys
+import urllib.request
+import json
+import time
+import uuid
+import random
+
+from logging import INFO
+
+from ssl import wrap_socket, CERT_NONE, CERT_REQUIRED, SSLError
+from ssl import match_hostname, CertificateError
+
+
+class ErrorCode:
+    """
+    We use some random numbers for error codes to make unit testing a little
+    easier, and for greppability
+    """
+    GENERIC_SSL_FAILURE = 42218
+    CN_MISMATCH = 34624
+    FILE_NOT_FOUND = 17253
+
+
+class LumberjackClient(object):
+    """Single threaded Lumberjack client to send events"""
+
+    # The sequence number is 32 bits, so it should rollover to 0 when it gets there
+    MAX_SEQUENCE_NUMBER = 2**32-1
+
+    def __init__(self, host, port, ca_certs=None, message_type="logstash-e2e"):
+        """
+        Specify the host, port, and file path to the concatenated list of CAs
+        """
+        self.host = host
+        self.port = port
+        self.ca_certs = ca_certs
+        self.sequence_number = 0
+        self.message_type = message_type
+        self.sock = None
+
+        # immediately send a packet setting the windows size to something huge
+        # since we don't care about ACKs
+        self.send(struct.pack(b"!ssI", b"1", b"W", self.MAX_SEQUENCE_NUMBER))
+
+    def format_message(self, record):
+        """Create a dict from the logging record"""
+        message = {
+            '@version': 1,
+            'message': record['message'],
+
+            # Extra Fields
+            'levelname': record['levelname'],
+            'logger': record['name'],
+            'type': self.message_type,
+            'path': '',
+
+            # required for logstash forwarder to work
+            'host': self.host,
+            "line": 0,
+            "offset": 0,  # this is completely meaningless, but required
+            "request_id": record['request_id']
+        }
+
+        return message
+
+    def make_socket(self, timeout=1):
+        """Make the socket and wrap it with SSL. A valid certificate is required from a trusted CA"""
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if hasattr(s, 'settimeout'):
+            s.settimeout(timeout)
+        s.connect((self.host, self.port))
+
+        try:
+            if self.host != "127.0.0.1" and self.ca_certs:
+                s = wrap_socket(s, cert_reqs=CERT_REQUIRED, ca_certs=self.ca_certs)
+            else:
+                s = wrap_socket(s, cert_reqs=CERT_NONE)
+        except SSLError as e:
+            # the ssl certificate was probably bad
+            # since logging doesn't work, print to stderr
+            traceback.print_exc()
+            print("ERROR %d: This is likely caused by a CN mismatch, an invalid certificate file at %s, or a verification failure based on your CA file" % (ErrorCode.GENERIC_SSL_FAILURE, self.ca_certs), file=sys.stderr)
+            s.close()
+            raise
+        except OSError as e:
+            s.close()
+            # if we get an ENOENT, that's because the ca_file is bad
+            if e.errno == errno.ENOENT:
+                traceback.print_exc()
+                print("ERROR %d: This error is probably caused by a bad CA file at %s" % (ErrorCode.FILE_NOT_FOUND, self.ca_certs), file=sys.stderr)
+            s.close()
+            raise
+
+        # some python versions do not do a CN match check, so we have to do it
+        try:
+            if self.host != "127.0.0.1":
+                # only check if this isn't executed from logstash server
+                match_hostname(s.getpeercert(), self.host)
+        except CertificateError as e:
+            traceback.print_exc()
+            print("ERROR %d: The CN provided by the server didn't match the host you connected to" % ErrorCode.CN_MISMATCH, file=sys.stderr)
+            s.close()
+            raise socket.error("CN Mismatch")
+
+        return s
+
+    def send(self, s):
+        """
+        Send a pickled string to the socket.
+
+        This function allows for partial sends which can happen when the
+        network is busy.
+        """
+        if self.sock is None:
+            self.sock = self.make_socket()
+        try:
+            if hasattr(self.sock, "sendall"):
+                self.sock.sendall(s)
+            else:
+                sentsofar = 0
+                left = len(s)
+                while left > 0:
+                    sent = self.sock.send(s[sentsofar:])
+                    sentsofar = sentsofar + sent
+                    left = left - sent
+        except socket.error:
+            self.sock.close()
+            self.sock = None  # so we can call make_socket next time
+
+    def emit(self, record):
+        """Format the record, and turn it into a logstash compatible packet"""
+        packet = self.packetize(self.format_message(record), self.sequence_number)
+        # critical section here for incremeting the sequence_number. Notice, we
+        # don't have to wrap a lock around this, as this is single threaded
+        self.sequence_number = self.sequence_number + 1 if self.sequence_number < self.MAX_SEQUENCE_NUMBER else 0
+        self.send(packet)
+
+    def packetize(self, data, sequence_number):
+        """
+        Turn a dict into a byte stream that is compatible with logstash-forwarder's protocol
+
+        The packet looks like this:
+
+        Header "1D" - the version and frame type (data) in ascii
+        32 bit unsigned, big-endian, integer sequence number
+        32 bit unsigned, big-endian, integer count of the number of (key, value) pairs
+        For each key, value pair:
+            32 bit unsigned, big-endian, integer number for the key length
+            the key (raw bytes)
+            32 bit unsigned, big-endian, integer number for the value length
+            the value (raw bytes)
+
+
+        The data *MUST* include a key/value pair for "host", "line" and
+        "offset". That fact isn't documented anywhere but here. You're welcome.
+        """
+        format_string = [b"!ssII"]
+        string = [b"1", b"D", sequence_number, len(data)]
+
+        for key, value in data.items():
+            # encode the keys and values as utf8-encoded bytestrings
+            key = key.encode('utf-8')
+            value = str(value).encode('utf-8')
+
+            # tack on the key length
+            format_string.append(b"I")
+            string.append(len(key))
+            # tack on the key itself
+            format_string.append(("%ds" % (len(key))).encode('utf-8'))
+            string.append(key)
+            # tack on the value length
+            format_string.append(b"I")
+            string.append(len(value))
+            # tack on the value
+            format_string.append(("%ds" % (len(value))).encode('utf-8'))
+            string.append(value)
+
+        return struct.pack(b"".join(format_string), *string)
+
+
+def main(args):
+    req_id = str(uuid.uuid4())
+    c = LumberjackClient(args.host, args.port, ca_certs=args.ca_certs,
+                         message_type=args.message_type)
+    c.emit(dict(name="logstash-e2e", levelname=INFO,
+                message='testing logstash e2e {}'.format(req_id),
+                request_id=req_id))
+    ts = time.time()
+
+    index = "logstash-%s" % datetime.datetime.utcnow().strftime("%Y.%m.%d")
+    get_body = {"query": {"match": {"request_id": req_id}},
+                "sort": [{"_timestamp": {"order": "desc"}}]
+                }
+
+    ok = False
+    elapsed = 0
+    retry = 0
+    es_hosts = args.es_host_port.split(',')
+    while not ok or elapsed >= args.critical:
+        es_host = es_hosts[random.randint(0, len(es_hosts)-1)]
+        try:
+            resp = urllib.request.urlopen("http://%s/%s/_search?size=1" % (es_host, index),
+                                  data=json.dumps(get_body).encode('utf-8'),
+                                  timeout=args.es_timeout)
+        except Exception as e:
+            print("Elasticsearch unreachable: %s" % str(e))
+            return 2
+        results = json.loads(resp.read().decode('utf-8'))
+        elapsed = time.time()-ts
+        for result in results['hits']['hits']:
+            if 'request_id' in result['_source'] and req_id == result['_source']['request_id']:
+                ok = True
+                break
+        if not ok:
+            # trivial retry with backoff
+            time.sleep((retry**0.5)+(random.randint(0, 1000) / 1000))
+            retry = retry + 1
+
+    with open(args.e2e_file, 'w') as fd:
+        if not ok or elapsed >= args.critical:
+            fd.write("ERROR: Taking longer than %ss to reach ES, aborting\n" % (args.critical,))
+        elif elapsed >= args.warn and elapsed < args.critical:
+            fd.write("WARNING: Took longer than %ss to reach ES\n" % (args.warn,))
+        else:
+            fd.write("OK: took: %s\n" % (elapsed,))
+        fd.flush()
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-H', dest="host", default="127.0.0.1")
+    parser.add_argument('-p', dest="port", default=5043, type=int)
+    parser.add_argument('--ca-certs', dest="ca_certs", default=None)
+    parser.add_argument('-m', dest="message_type", default="logstash-e2e")
+    parser.add_argument('-w', dest="warn", type=int, default=60)
+    parser.add_argument('-c', dest="critical", type=int, default=120)
+    parser.add_argument('--es', dest="es_host_port", required=True,
+                        help="Comma separated list of elasticsearch " + \
+                        "host:port, e.g: host:port,host1:port1..")
+    parser.add_argument('-t', dest="es_timeout", type=int, default=5,
+                        help="timeout for ES http request")
+    parser.add_argument('-f', dest="e2e_file", required=True,
+                        help="path to the output file")
+    args = parser.parse_args()
+    try:
+        sys.exit(main(args))
+    except Exception as e:
+        with open(args.e2e_file, 'w') as fd:
+            fd.write("ERROR: Unhandled error: %s\n" % (str(e),))
+            fd.flush()
+        traceback.print_exc()
+        sys.exit(1)

=== modified file 'files/upstart/logstash-indexer.conf'
--- files/upstart/logstash-indexer.conf	2014-09-23 12:11:08 +0000
+++ files/upstart/logstash-indexer.conf	2016-11-23 22:25:27 +0000
@@ -11,6 +11,7 @@
 respawn
 respawn limit 5 30
 env HOME={{BASEPATH}}
+env LS_HEAP_SIZE={{HEAP_SIZE}}
 chdir {{BASEPATH}}
 setuid logstash
 setgid adm
@@ -18,4 +19,4 @@
 
 script
     {{BASEPATH}}/bin/logstash agent -f {{BASEPATH}}/conf.d/
-end script
\ No newline at end of file
+end script

=== modified file 'hooks/client-relation-changed'
--- hooks/client-relation-changed	2015-09-08 19:45:13 +0000
+++ hooks/client-relation-changed	2016-11-23 22:25:27 +0000
@@ -1,6 +1,9 @@
 #!/usr/bin/python
 
+import json
 import os
+import shlex
+import subprocess
 import sys
 
 sys.path.insert(0, os.path.join(os.environ['CHARM_DIR'], 'lib'))
@@ -19,45 +22,26 @@
 BASEPATH = os.path.join(os.path.sep, 'opt', 'logstash')
 
 
[e-mail]('client-relation-changed')
[e-mail]('client-relation-changed', 'client-relation-departed')
 def rest_changed():
-    cache_hosts()
     write_config()
     host.service_restart(SERVICE) or host.service_start(SERVICE)
 
 
 def write_config():
-    with open('host_cache', 'r') as f:
-        hosts = f.readlines()
-    if not hosts:
-        sys.exit(0)
-
-    # Use last host in list as it will be the most recently added
-    # and first host in list may not exist anymore! TODO fix that.
-    opts = {'hosts': hosts[-1].rstrip()}
-
-    out = os.path.join(BASEPATH, 'conf.d', 'output-elasticsearch.conf')
-    with open(out, 'w') as p:
-        p.write(render(os.path.basename(out), opts))
-
-
-def cache_hosts():
     rels = hookenv.relations_of_type("client")
     if not rels:
         log('No client relations. Assuming nothing to do.')
         sys.exit(0)
-    if not os.path.exists('host_cache'):
-        open('host_cache', 'a').close()
-    for rel in rels:
-        host = rel.get('host')
-        if not host:
-            log('No host received for relation: {}.'.format(rel))
-            continue
-        with open('host_cache', 'r') as f:
-            hosts = f.readlines()
-        if host not in hosts:
-            with open('host_cache', 'a') as f:
-                f.write('{}\n'.format(host))
+    hosts = json.dumps(
+        ["%s:%s" % (client['host'], client['port']) for client in rels
+            if 'host' in client])
+
+    out = os.path.join(BASEPATH, 'conf.d', 'output-elasticsearch.conf')
+    with open(out, 'w') as p:
+        p.write(render(os.path.basename(out), dict(hosts=hosts)))
+    # update nrpe now that we have client relations
+    subprocess.check_output(shlex.split('hooks/nrpe-external-master-relation-changed'))
 
 
 if __name__ == "__main__":

=== added symlink 'hooks/client-relation-departed'
=== target is u'client-relation-changed'
=== modified file 'hooks/config-changed'
--- hooks/config-changed	2015-09-08 19:05:34 +0000
+++ hooks/config-changed	2016-11-23 22:25:27 +0000
@@ -38,15 +38,20 @@
 
     # This only actually opens the port if we've exposed the service in juju
     hookenv.open_port(5043)
-    hookenv.open_port(5959, protocol='UDP')
-    hookenv.open_port(5959, protocol='TCP')
-
-    # The install hook is idempotent, so re-run it.
-    # XXX: no, it's not idempotent. it fails if the tarball was already
-    # extracted
-    # subprocess.check_output(shlex.split('hooks/install'))
-
-    # Restart the service when configuration has changed.
+
+    # Verify the config to get a decent error message for bad configuration.
+    try:
+        subprocess.check_output([
+            '/opt/logstash/bin/logstash', 'agent', '-f',
+            '/opt/logstash/conf.d/', '--configtest'
+        ])
+    except subprocess.CalledProcessError, e:
+        # The exception doesn't print the output, by default.
+        print(e.output)
+        raise
+
+    # Restart the service when configuration has changed
+    # and the config is valid.
     subprocess.check_output(shlex.split('hooks/start'))
 
     subprocess.check_output(shlex.split('hooks/nrpe-external-master-relation-changed'))
@@ -61,6 +66,8 @@
     key_file = os.path.join(cert_dir, 'logstash.key')
 
     for f in files:
+        if not f.endswith(".conf"):
+            continue
         # skip output-elasticsearch.conf, is managed by
         # hooks/client-relation-changed
         if os.path.basename(f) == "output-elasticsearch.conf":
@@ -75,6 +82,12 @@
     if config_data['extra-config']:
         with open(os.path.join(BASEPATH, 'conf.d', 'extra.conf'), 'w') as f:
             f.write(str(base64.b64decode(config_data['extra-config'])))
+    if config_data['extra-patterns']:
+        patterns_dir = os.path.join(BASEPATH, 'patterns')
+        if not os.path.exists(patterns_dir):
+            os.mkdir(patterns_dir)
+        with open(os.path.join(patterns_dir, 'extra-patterns'), 'w') as f:
+            f.write(str(base64.b64decode(config_data['extra-patterns'])))
 
     # Only setup lumberjack protocol if ssl cert and key are configured
     if config_data['ssl_cert'] and config_data['ssl_key']:
@@ -100,7 +113,11 @@
 def place_upstart_template():
     out = os.path.join(os.path.sep, 'etc', 'init', '{}.conf'.format(SERVICE))
     templ = os.path.join('files', 'upstart')
-    opts = {'BASEPATH': BASEPATH}
+    config_data = hookenv.config()
+    opts = {
+        'BASEPATH': BASEPATH,
+        'HEAP_SIZE': config_data['heap_size'],
+    }
 
     with open(out, 'w') as p:
         p.write(render('{}.conf'.format(SERVICE), opts, template_dir=templ))

=== modified file 'hooks/nrpe-external-master-relation-changed'
--- hooks/nrpe-external-master-relation-changed	2015-09-29 19:43:29 +0000
+++ hooks/nrpe-external-master-relation-changed	2016-11-23 22:25:27 +0000
@@ -1,13 +1,14 @@
 #!/usr/bin/python
 
+import glob
 import os
+import shutil
 import sys
 
 sys.path.insert(0, os.path.join(os.environ['CHARM_DIR'], 'lib'))
 
-from charmhelpers.core import hookenv
+from charmhelpers.core import hookenv, templating
 from charmhelpers.contrib.charmsupport import nrpe
-from charmhelpers.contrib.charmsupport.nrpe import NRPE
 
 hooks = hookenv.Hooks()
 log = hookenv.log
@@ -47,8 +48,21 @@
         self.service_template = CustomIntervalCheck.service_template % intervals_config
 
 
+def install_nrpe_scripts():
+    # copy the scripts
+    scripts_src = os.path.join(os.environ["CHARM_DIR"], "files",
+                               "nrpe")
+    scripts_dst = "/usr/lib/nagios/plugins"
+    if not os.path.exists(scripts_dst):
+        os.makedirs(scripts_dst)
+    for fname in glob.glob(os.path.join(scripts_src, "*.py")):
+        shutil.copy2(fname,
+                     os.path.join(scripts_dst, os.path.basename(fname)))
+
+
 @hooks.hook('nrpe-external-master-relation-changed')
 def update_nrpe_checks():
+    install_nrpe_scripts()
     nrpe_compat = nrpe.NRPE()
     conf = nrpe_compat.config
     check_procs_params = conf.get('nagios_check_procs_params')
@@ -60,6 +74,8 @@
         )
     check_tcp_params = conf.get('nagios_check_tcp_params')
     check_cert_params = conf.get('nagios_check_cert_params')
+    check_e2e_params = conf.get('nagios_check_e2e_params')
+    check_e2e_cron = conf.get('nagios_check_e2e_cron')
     config_data = hookenv.config()
     # Only setup lumberjack protocol if ssl cert and key are configured
     if config_data['ssl_cert'] and config_data['ssl_key']:
@@ -79,6 +95,31 @@
                 retry_check_interval=120,  # minutes
             )
             nrpe_compat.checks.append(cert_check)
+        # add e2e check using lumberjack protocol
+        rels = hookenv.relations_of_type("client")
+        if rels and check_e2e_cron:
+            # delete old check
+            if os.path.exists('/etc/nagios/nrpe.d/check_e2e-check.cfg'):
+                os.unlink('/etc/nagios/nrpe.d/check_e2e-check.cfg')
+            es_ips = ','.join(["%s:%s" % (client['host'], client['port'])
+                               for client in rels if 'host' in client])
+            e2e_output_file = "/var/lib/nagios/logstash-e2e-check.output"
+            charm_dir = hookenv.charm_dir()
+            # write cronjob to run e2e script every 5mins
+            context = {'es_ips': es_ips,
+                       'e2e_file': e2e_output_file,
+                       'e2e_params': check_e2e_params,
+                       'cron_expression': check_e2e_cron,
+                       'charm_dir': charm_dir}
+            templating.render('e2e-cron.tmpl', '/etc/cron.d/e2e-check',
+                              context=context, perms=0554)
+            nrpe_compat.add_check(
+                shortname='e2e_output',
+                description='Check E2E script output',
+                check_cmd='check_e2e_output.py -f {}'.format(e2e_output_file)
+            )
+        else:
+            log('No client relations. Not adding e2e check.')
 
     nrpe_compat.write()
 

=== added file 'templates/e2e-cron.tmpl'
--- templates/e2e-cron.tmpl	1970-01-01 00:00:00 +0000
+++ templates/e2e-cron.tmpl	2016-11-23 22:25:27 +0000
@@ -0,0 +1,4 @@
+# Cron job to run e2e check 
+MAILTO=""
+{{ cron_expression }} nagios run-this-one python3 {{ charm_dir }}/files/nrpe/check_lumberjack_e2e.py --es {{ es_ips }} -f {{ e2e_file }} {{ e2e_params }}
+

=== modified file 'templates/input-file-syslog.conf'
--- templates/input-file-syslog.conf	2014-09-23 12:12:45 +0000
+++ templates/input-file-syslog.conf	2016-11-23 22:25:27 +0000
@@ -1,1 +1,1 @@
-input {  file { type => 'syslog' path => ['/var/log/syslog','/var/log/kern.log','/var/log/auth.log','/var/log/dpkg.log'] sincedb_path => '{{BASEPATH}}' } } 
\ No newline at end of file
+input {  file { type => 'syslog' path => ['/var/log/syslog','/var/log/kern.log','/var/log/auth.log','/var/log/dpkg.log'] } }

=== removed file 'templates/input-python-logging.conf'
--- templates/input-python-logging.conf	2015-06-30 16:42:53 +0000
+++ templates/input-python-logging.conf	1970-01-01 00:00:00 +0000
@@ -1,14 +0,0 @@
-# Input for python-logstash [[https://pypi.python.org/pypi/python-logstash]]
-# via udp:5959.
-input {
-  udp {
-    # The port to listen on.
-    port => 5959
-
-    # Receives plain JSON objects.
-    codec => "json"
-
-    # Just an identifier, change it to whatever makes more sense.
-    type => "python-logstash"
-  }
-}

=== modified file 'templates/input-redis.conf'
--- templates/input-redis.conf	2014-09-23 12:12:45 +0000
+++ templates/input-redis.conf	2016-11-23 22:25:27 +0000
@@ -1,1 +1,1 @@
-input { redis { host => "127.0.0.1" data_type => "list" type => "redis" tags => ["redis"] key => "logstash" message_format => "json_event" } }
\ No newline at end of file
+input { redis { host => "127.0.0.1" data_type => "list" type => "redis" tags => ["redis"] key => "logstash" } }

=== modified file 'templates/output-elasticsearch.conf'
--- templates/output-elasticsearch.conf	2014-09-23 16:59:13 +0000
+++ templates/output-elasticsearch.conf	2016-11-23 22:25:27 +0000
@@ -1,5 +1,5 @@
 output {
-    elasticsearch_http { 
-        host => "{{hosts}}"
+    elasticsearch {
+        hosts => {{ hosts }}
     }
 }