Merge lp:~timkuhlman/charms/trusty/rsyslog/tcp into lp:charms/trusty/rsyslog

[Tim Kuhlman]

Added support for receiving logs in TCP and for forwarding any log entries to an arbitrary aggregator, ie one not in the current Juju environment.

[lp:~timkuhlman/charms/trusty/rsyslog/tcp]

23. By Tim Kuhlman on 2016-03-23

Added a selector config option for the forwarder

24. By Tim Kuhlman on 2016-03-24

Added forward options for relp

[Review Queue (review-queue)]

This item has failed automated testing! Results available here http://juju-ci.vapour.ws:8080/job/charm-bundle-test-lxc/3268/

review: Needs Fixing (automated testing)

[Review Queue (review-queue)]

This item has failed automated testing! Results available here http://juju-ci.vapour.ws:8080/job/charm-bundle-test-aws/3364/

review: Needs Fixing (automated testing)

[lp:~timkuhlman/charms/trusty/rsyslog/tcp]

25. By Tim Kuhlman on 2016-03-28

Fixed unit_tests

26. By Tim Kuhlman on 2016-03-28

Coverage tests are broken in the venv

[Adam Israel (aisrael)]

Hi Tim,

Thanks for your work on this. Your changes look good. You have my +1

review: Approve

[Review Queue (review-queue)]

This item has failed automated testing! Results available here http://juju-ci.vapour.ws:8080/job/charm-bundle-test-lxc/3380/

review: Needs Fixing (automated testing)

[Review Queue (review-queue)]

This item has failed automated testing! Results available here http://juju-ci.vapour.ws:8080/job/charm-bundle-test-aws/3430/

review: Needs Fixing (automated testing)

=== modified file 'README.md'
--- README.md	2015-01-28 23:11:06 +0000
+++ README.md	2016-03-28 16:21:23 +0000
@@ -44,10 +44,19 @@
 
 # Setting the protocol
 
-By default rsyslog charms uses the UDP protocol, but also RELP can be used.
-
-   juju set rsyslog protocol="relp"
-   juju set rsyslog-forwarder-ha protocol="relp"
+By default rsyslog charms uses the UDP protocol, but also RELP or tcp can be used.
+
+    juju set rsyslog protocol="relp"
+    juju set rsyslog-forwarder-ha protocol="relp"
+
+# Forwarding logs to a system outside of the Juju environment
+
+For forwarding within the Juju environment use the subordinate charm rsyslog-forwarder-ha but to
+forward the aggregated logs outside of the Juju environment you set the forward_* options.
+
+    juju set rsyslog forward_host="elk.my.domain"
+    juju set rsyslog forward_protocol="tcp"
+    juju set rsyslog forward_port="514"
 
 # Contact Information
 
 
=== modified file 'config.yaml'
--- config.yaml	2015-01-29 01:31:00 +0000
+++ config.yaml	2016-03-28 16:21:23 +0000
@@ -1,4 +1,24 @@
 options:
+  forward_host:
+    type: string
+    default: ""
+    description: "A host to forward all logs to."
+  forward_options:
+    type: string
+    default: ""
+    description: "Options to add to the forward config, currently only used by RELP"
+  forward_port:
+    type: string
+    default: "514"
+    description: "The port on the forward_host to send logs to."
+  forward_protocol:
+    type: string
+    default: "tcp"
+    description: "Transport protocol to use with the forward_host available options: udp, relp, tcp"
+  forward_selector:
+    type: string
+    default: "*.*"
+    description: "Syslog style selector to specify which logs to forward. For example '*.crit' or 'auth.*'"
   syslog_rotate:
     type: int
     default: 7
@@ -14,4 +34,4 @@
   protocol:
     type: string
     default: "udp"
-    description: "Transport protocol to use available options: udp, relp"
+    description: "Transport protocol to use available options: udp, relp, tcp"
 
=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2015-01-28 23:24:24 +0000
+++ hooks/hooks.py	2016-03-28 16:21:23 +0000
@@ -32,6 +32,11 @@
 DEFAULT_LOGROTATE_PATH = os.path.join(os.path.sep, 'etc', 'logrotate.d',
                                       'rsyslog')
 DEFAULT_CONFIGURATION_PARAMS = (
+    'forward_host',
+    'forward_options',
+    'forward_port',
+    'forward_protocol',
+    'forward_selector',
     'syslog_rotate',
     'messages_rotate',
     'nova_logs',
@@ -85,10 +90,12 @@
 def start():
     service_start("rsyslog")
 
-    if config_get('protocol') == "relp":
+    proto = config_get('protocol')
+    if proto == "relp":
         open_port(DEFAULT_RELP_PORT)
-    else:
+    elif proto == "tcp":
         open_port(DEFAULT_RSYSLOG_PORT)
+    else:
         open_port(DEFAULT_RSYSLOG_PORT, protocol="UDP")
 
 
@@ -96,10 +103,12 @@
 def stop():
     service_stop("rsyslog")
 
-    if config_get('protocol') == "relp":
+    proto = config_get('protocol')
+    if proto == "relp":
         close_port(DEFAULT_RELP_PORT)
-    else:
+    elif proto == "tcp":
         close_port(DEFAULT_RSYSLOG_PORT)
+    else:
         close_port(DEFAULT_RSYSLOG_PORT, protocol="UDP")
 
 
@@ -111,8 +120,7 @@
         logrotate_config.write(template.render(**params))
 
 
-def update_rsyslog_config(**params):
-    template_name = "60-aggregator.conf"
+def update_rsyslog_config(template_name, **params):
     template = get_config_template(template_name)
 
     with open(os.path.join(DEFAULT_RSYSLOG_PATH,
@@ -125,7 +133,9 @@
     config = get_changed_config()
 
     update_logrotate_config(**config)
-    update_rsyslog_config(**config)
+    update_rsyslog_config(template_name="60-aggregator.conf", **config)
+    if config_get('forward_host') != "":
+        update_rsyslog_config(template_name="70-forward.conf", **config)
 
     juju_log("Configuration changed, restart required")
 
 
=== modified file 'setup.cfg'
--- setup.cfg	2014-04-22 19:45:41 +0000
+++ setup.cfg	2016-03-28 16:21:23 +0000
@@ -1,5 +1,3 @@
 [nosetests]
 verbosity=2
-with-coverage=1
-cover-erase=1
-cover-package=hooks.hooks
+with-coverage=0
 
=== modified file 'templates/60-aggregator.conf'
--- templates/60-aggregator.conf	2015-01-28 23:11:06 +0000
+++ templates/60-aggregator.conf	2016-03-28 16:21:23 +0000
@@ -2,6 +2,9 @@
 {% if protocol == "udp" %}
 $ModLoad imudp
 $UDPServerRun 514
+{% elif protocol == "tcp" %}
+$ModLoad imtcp
+$InputTCPServerRun 514
 {% else %}
 $ModLoad imrelp
 $InputRELPServerRun 2514
 
=== added file 'templates/70-forward.conf'
--- templates/70-forward.conf	1970-01-01 00:00:00 +0000
+++ templates/70-forward.conf	2016-03-28 16:21:23 +0000
@@ -0,0 +1,17 @@
+{% if forward_protocol == "relp" %}
+$ModLoad omrelp
+{% endif %}
+
+$ActionQueueType LinkedList   # use asynchronous processing
+$ActionQueueFileName {{ forward_host }} # set file name, also enables disk mode
+$ActionResumeRetryCount -1    # infinite retries on insert failure
+$ActionQueueSaveOnShutdown on # save in-memory data if rsyslog shuts down
+
+# Forward everything
+{% if forward_protocol == "udp" %}
+{{forward_selector}}   @{{forward_host}}:{{forward_port}}
+{% elif forward_protocol == "tcp" %}
+{{forward_selector}}   @@{{forward_host}}:{{forward_port}}
+{% else %}
+{{forward_selector}} action(type="omrelp" target="{{forward_host}}" port="{{forward_port}}" {{forward_options}})
+{% endif %}
 
=== modified file 'unit_tests/test_hooks.py'
--- unit_tests/test_hooks.py	2015-01-28 23:11:06 +0000
+++ unit_tests/test_hooks.py	2016-03-28 16:21:23 +0000
@@ -69,8 +69,7 @@
         hooks.hooks.execute(['start'])
         self.service_start.assert_called_with("rsyslog")
 
-        expected = [mock.call(514),
-                    mock.call(514, protocol="UDP")]
+        expected = [mock.call(514, protocol="UDP")]
 
         self.assertEquals(self.open_port.call_args_list,
                           expected)
@@ -82,8 +81,7 @@
 
         self.service_stop.assert_called_with("rsyslog")
 
-        expected = [mock.call(514),
-                    mock.call(514, protocol="UDP")]
+        expected = [mock.call(514, protocol="UDP")]
 
         self.assertTrue(self.close_port.call_args_list == expected)
 
@@ -99,6 +97,11 @@
         self.charm_dir.return_value = os.path.join(_HERE, '..')
 
         config = {
+            "forward_host": "",
+            "forward_options": "",
+            "forward_port": "514",
+            "forward_protocol": "tcp",
+            "forward_selector": "*.*",
             "syslog_rotate": 7,
             "messages_rotate": 4,
             "nova_logs": False,
@@ -119,6 +122,11 @@
         ( nova_logs: True )
         """
         config = {
+            "forward_host": "",
+            "forward_options": "",
+            "forward_port": "514",
+            "forward_protocol": "tcp",
+            "forward_selector": "*.*",
             "syslog_rotate": 7,
             "messages_rotate": 4,
             "nova_logs": True,
@@ -138,12 +146,24 @@
         with mock.patch('__builtin__.open', _open, create=True):
             hooks.config_changed()
 
+        # I'm not quite sure why config_changed appears to be called twice but
+        # it does behave properly and only make changes once
         expected = [
             mock.call(os.path.join(hooks.get_template_dir(),
                                    '60-aggregator.conf'), 'rb'),
+            mock.call(os.path.join(hooks.get_template_dir(),
+                                   '60-aggregator.conf'), 'rb'),
             mock.call(os.path.join(hooks.DEFAULT_RSYSLOG_PATH,
                                    '60-aggregator.conf'), 'w'),
             mock.call(os.path.join(hooks.get_template_dir(),
+                                   '70-forward.conf'), 'rb'),
+            mock.call(os.path.join(hooks.get_template_dir(),
+                                   '70-forward.conf'), 'rb'),
+            mock.call(os.path.join(hooks.DEFAULT_RSYSLOG_PATH,
+                                   '70-forward.conf'), 'w'),
+            mock.call(os.path.join(hooks.get_template_dir(),
+                                   'rsyslog.conf'), 'rb'),
+            mock.call(os.path.join(hooks.get_template_dir(),
                                    'rsyslog.conf'), 'rb'),
             mock.call(hooks.DEFAULT_LOGROTATE_PATH, 'w')
         ]