Merge lp:~verterok/charms/trusty/apache2/tanuki_merge-upstream-r72 into lp:~tanuki/charms/trusty/apache2/trunk

[Guillermo Gonzalez]

merge upstream trunk r72

merge upstream trunk r72

[Guillermo Gonzalez (verterok)]

trivial

review: Approve

=== modified file 'Makefile'
--- Makefile	2015-03-10 10:56:19 +0000
+++ Makefile	2016-04-26 22:51:55 +0000
@@ -17,8 +17,16 @@
 	@(charm proof $(PWD) || [ $$? -eq 100 ]) && echo OK
 	@test `cat revision` = 0 && rm revision
 
-.venv:
-	sudo apt-get install -y python-apt python-virtualenv python-jinja2
+/usr/bin/apt:
+	sudo apt-get install -y python-apt
+
+/usr/bin/virtualenv:
+	sudo apt-get install -y python-virtualenv
+
+/usr/lib/python2.7/dist-packages/jinja2:
+	sudo apt-get install -y python-jinja2
+
+.venv: /usr/bin/apt /usr/bin/virtualenv /usr/lib/python2.7/dist-packages/jinja2
 	virtualenv .venv --system-site-packages
 	.venv/bin/pip install -I nose testtools mock pyyaml
 
@@ -28,8 +36,8 @@
 
 lint:
 	@echo Checking for Python syntax...
-	@flake8 $(HOOKS_DIR) --ignore=E123 --exclude=$(HOOKS_DIR)/charmhelpers && echo hooks OK
-	@flake8 tests --ignore=E123 && echo tests OK
+	@flake8 $(HOOKS_DIR) --ignore=E123,E402 --exclude=$(HOOKS_DIR)/charmhelpers && echo hooks OK
+	@flake8 tests --ignore=E123,E402 && echo tests OK
 
 sourcedeps: $(PWD)/config-manager.txt
 	@echo Updating source dependencies...
@@ -42,4 +50,7 @@
 		-d hooks/charmhelpers
 	@echo Do not forget to commit the updated files if any.
 
-.PHONY: revision proof .venv test lint sourcedeps charm-payload
+clean:
+	rm -rf .venv
+
+.PHONY: revision proof test lint sourcedeps charm-payload

=== modified file 'README.md'
--- README.md	2015-04-15 22:59:59 +0000
+++ README.md	2016-04-26 22:51:55 +0000
@@ -265,6 +265,12 @@
 
  * `ports` - A space-separated list of ports that the site uses.
 
+### Using the logs relation
+
+The logs relation is for use with a logging subordinate charm. The beaver
+subordinate can be deployed and related to apache and logstash. Beaver will
+tail apache logs and send the logs to logstash.
+
 ## Certs, keys and chains
 
 `ssl_keylocation`, `ssl_certlocation` and `ssl_chainlocation` are

=== modified file 'config.yaml'
--- config.yaml	2015-08-28 12:47:41 +0000
+++ config.yaml	2016-04-26 22:51:55 +0000
@@ -206,3 +206,16 @@
     type: string
     description: Comma seperated list of OpenID providers for authentication.
     default: ""
+  apt-key-id:
+    type: string
+    default: ""
+    description: A PGP key id.  This is used with PPA and the source
+      option to import a PGP public key for verifying repository signatures.
+      This value must match the PPA for apt-source.
+  apt-source:
+    type: string
+    default: ""
+    description: From where to install packages. This is the PPA source line.
+      Note that due to a bug in software-properties add-apt-repository cannot
+        add the ondrej/apache2 ppa, so the default value here is a full
+        sources line.

=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2015-09-23 13:57:41 +0000
+++ hooks/hooks.py	2016-04-26 22:51:55 +0000
@@ -27,7 +27,7 @@
     unit_get
 )
 from charmhelpers.contrib.charmsupport import nrpe
-from charmhelpers.fetch import apt_update
+from charmhelpers.fetch import apt_update, add_source
 
 ###############################################################################
 # Global variables
@@ -337,6 +337,12 @@
 
 
 def install_hook():
+    apt_source = config_get('apt-source') or ''
+    apt_key_id = config_get('apt-key-id') or False
+    if apt_source and apt_key_id:
+        print apt_source + " and " + apt_key_id
+        add_source(apt_source, apt_key_id)
+        open('config.apt-source', 'w').write(apt_source)
     if not os.path.exists(default_apache2_service_config_dir):
         os.mkdir(default_apache2_service_config_dir, 0600)
     apt_update(fatal=True)
@@ -349,6 +355,15 @@
         ensure_package_status(service_affecting_packages,
                               config_get('package_status'))
     ensure_extra_packages()
+    # the apache2 deb does not yet have http2 module in mods-available. Add it.
+    open('/etc/apache2/mods-available/http2.load', 'w').write(
+        'LoadModule http2_module /usr/lib/apache2/modules/mod_http2.so')
+    open('/etc/apache2/mods-available/http2.conf', 'w').write(
+        '''<IfModule http2_module>
+  ProtocolsHonorOrder On
+  Protocols h2 http/1.1
+</IfModule>
+''')
     return install_status
 
 
@@ -357,7 +372,8 @@
     if extra:
         install_status = apt_get_install(extra)
         if install_status == 0:
-            ensure_package_status(extra, config_get('package_status'))
+            ensure_package_status(filter(None, extra.split(' ')),
+                                  config_get('package_status'))
 
 
 def dump_data(data2dump, log_prefix):
@@ -642,6 +658,18 @@
     relationship_data = {}
     config_data = config_get()
 
+    apt_source = config_data['apt-source']
+    old_apt_source = ''
+    try:
+        old_apt_source = open('config.apt-source', 'r').read()
+    except IOError:
+        pass
+    if old_apt_source != apt_source:
+        subprocess.check_call(['add-apt-repository', '--yes', '-r',
+                               old_apt_source])
+        add_source(apt_source, config_data['apt-key-id'])
+        open('config.apt-source', 'w').write(apt_source)
+
     ensure_package_status(service_affecting_packages,
                           config_data['package_status'])
     ensure_extra_packages()
@@ -795,6 +823,8 @@
     update_nrpe_checks()
     update_etc_hosts()
     ship_logrotate_conf()
+    if config_get().changed('servername'):
+        logs_relation_joined()
 
 
 def ensure_disabled(sites):
@@ -1014,6 +1044,53 @@
         yaml.safe_dump(ports, pfile)
 
 
+def get_log_files():
+    """
+    Read all of the apache config files from __ and get ErrorLog and AccessLog
+    values.
+
+    Returns a tuple with first value list of access log files and second value
+    list of error log files.
+    """
+    access_logs = []
+    error_logs = []
+    for protocol in ['http', 'https']:
+        vhost_name = '%s_%s' % (config_get()['servername'], protocol)
+        vhost_file = site_filename(vhost_name)
+        try:
+            # Using read().split('\n') here to work around a mocks open_mock
+            # inadequacy: http://bugs.python.org/issue17467
+            for line in open(vhost_file, 'r').read().split('\n'):
+                if 'CustomLog' in line:
+                    access_logs.append(line.split()[1])
+                elif 'ErrorLog' in line:
+                    error_logs.append(line.split()[1])
+        except:
+            pass
+    return access_logs, error_logs
+
+
+def logs_relation_joined():
+    """
+    Sets relation value with filenames
+    """
+    access_log_files, error_log_files = get_log_files()
+    log_files = access_log_files[:]
+    log_files.extend(error_log_files)
+    types = ['apache_access' for a in access_log_files]
+    types.extend(['apache_error' for a in error_log_files])
+    data = {'files': '\n'.join(log_files),
+            'types': '\n'.join(types),
+            }
+    _relation_ids = relation_ids('logs')
+    for _relation_id in _relation_ids:
+        log("logs-relation-joined setting relation data for {} to {}".format(
+            _relation_id, data))
+        relation_set(
+            relation_id=_relation_id,
+            relation_settings=data)
+
+
 ###############################################################################
 # Main section
 ###############################################################################
@@ -1053,6 +1130,8 @@
         update_nrpe_checks()
     elif hook_name == "vhost-config-relation-changed":
         config_changed()
+    elif hook_name == "logs-relation-joined":
+        logs_relation_joined()
     else:
         print "Unknown hook"
         sys.exit(1)

=== modified file 'hooks/install'
--- hooks/install	2013-10-10 22:49:28 +0000
+++ hooks/install	2016-04-26 22:51:55 +0000
@@ -5,5 +5,8 @@
 juju-log 'Invoking charm-pre-install hooks'
 [ -d exec.d ] && ( for f in exec.d/*/charm-pre-install; do [ -x $f ] && /bin/sh -c "$f"; done )
 
+juju-log 'Ensuring python, python-apt and python-yaml are installed'
+apt-get install -y python python-apt python-yaml
+
 juju-log 'Invoking python-based install hook'
 python hooks/hooks.py install

=== added symlink 'hooks/logs-relation-joined'
=== target is u'hooks.py'
=== modified file 'hooks/tests/test_balancer_hook.py'
--- hooks/tests/test_balancer_hook.py	2014-12-29 22:59:20 +0000
+++ hooks/tests/test_balancer_hook.py	2016-04-26 22:51:55 +0000
@@ -497,6 +497,7 @@
         if os.path.exists(self.log_path):
             os.remove(self.log_path)
 
+    @patch('hooks.open')
     @patch('hooks.config_get')
     @patch('os.path.exists')
     @patch('os.mkdir')
@@ -504,7 +505,8 @@
     @patch('hooks.log', MagicMock())
     @patch('hooks.apt_update')
     def test_installs_hook(
-            self, apt_update, apt_get_install, mkdir, exists, config_get):
+            self, apt_update, apt_get_install, mkdir, exists, config_get,
+            open):
         exists.return_value = self.not_a_dir
         config_get.return_value = None
         apt_get_install.return_value = 'some result'
@@ -523,19 +525,22 @@
             call('apache2'),
         ])
 
-    @patch('hooks.config_get')
+    @patch('hooks.open')
     @patch('os.path.exists')
     @patch('os.mkdir')
     @patch('hooks.apt_get_install')
     @patch('hooks.log', MagicMock())
     @patch('hooks.apt_update')
     def test_install_hook_installs_extra_packages(
-            self, apt_update, apt_get_install, mkdir, exists, config_get):
+            self, apt_update, apt_get_install, mkdir, exists,
+            open):
         exists.return_value = self.dir_exists
-        config_get.return_value = "extra"
+        c = {'extra_packages': 'extra', 'apt-source': '', 'apt-key-id': ''}
+        config_get = MagicMock(wraps=c.get)
         apt_get_install.return_value = 'some result'
 
-        result = hooks.install_hook()
+        with patch('hooks.config_get', config_get):
+            result = hooks.install_hook()
 
         self.assertEqual(result, 'some result')
         apt_get_install.assert_has_calls([
@@ -547,6 +552,7 @@
             call('extra'),
         ])
 
+    @patch('hooks.open')
     @patch('hooks.config_get')
     @patch('os.path.exists')
     @patch('os.mkdir')
@@ -554,7 +560,8 @@
     @patch('hooks.log', MagicMock())
     @patch('hooks.apt_update')
     def test_doesnt_create_dir_to_install_hooks_if_not_needed(
-            self, apt_update, apt_get_install, mkdir, exists, config_get):
+            self, apt_update, apt_get_install, mkdir, exists, config_get,
+            open):
         exists.return_value = self.dir_exists
         config_get.return_value = None
         apt_get_install.return_value = 'some result'

=== modified file 'hooks/tests/test_cert.py'
--- hooks/tests/test_cert.py	2014-05-24 06:33:34 +0000
+++ hooks/tests/test_cert.py	2016-04-26 22:51:55 +0000
@@ -73,6 +73,12 @@
     def test_is_selfsigned_cert_stale_public_address_changed(
             self, unit_get, log):
         """With different servername, cert *should* be regenerated."""
+        try:
+            from pyasn1.codec.der import decoder  # noqa
+            from pyasn1_modules import rfc2459  # noqa
+        except ImportError:
+            # This test is utopic+ only
+            return
         config = {"servername": "servername"}
         cert_file = os.path.join(self.tempdir, "cert")
         key_file = os.path.join(self.tempdir, "key")
@@ -86,6 +92,12 @@
     def test_is_selfsigned_cert_stale_private_address_changed(
             self, unit_get, log):
         """With different servername, cert *should* be regenerated."""
+        try:
+            from pyasn1.codec.der import decoder  # noqa
+            from pyasn1_modules import rfc2459  # noqa
+        except ImportError:
+            # This test is utopic+ only
+            return
         config = {"servername": "servername"}
         cert_file = os.path.join(self.tempdir, "cert")
         key_file = os.path.join(self.tempdir, "key")

=== modified file 'hooks/tests/test_config_changed.py'
--- hooks/tests/test_config_changed.py	2015-09-23 13:45:57 +0000
+++ hooks/tests/test_config_changed.py	2016-04-26 22:51:55 +0000
@@ -4,6 +4,10 @@
 import shutil
 import tempfile
 import os
+import sys
+
+sys.path.insert(0, os.path.join(os.environ['CHARM_DIR'], 'lib'))
+from charmhelpers.core import hookenv
 
 
 class ConfigChangedTest(TestCase):
@@ -48,7 +52,7 @@
             mock_conf_disable, mock_open_port, mock_close_port, mock_call,
             mock_set_open_ports, mock_get_open_ports, mock_update_etc_hosts):
         """config-changed hook: Site directories should be empty."""
-        mock_config_get.return_value = {
+        mock_config_get.return_value = hookenv.Config({
             "ssl_cert": "",
             "ssl_key": "",
             "package_status": "",
@@ -64,7 +68,8 @@
             "servername": "foobar",
             "vhost_http_template": "",
             "vhost_https_template": "",
-        }
+            "apt-source": "",
+        })
         base = patch.object(hooks, 'default_apache_base_dir', self.dirname)
         config22 = patch.object(hooks, 'default_apache22_config_dir',
                                 "%s/conf.d" % self.dirname)

=== added file 'hooks/tests/test_log_relation.py'
--- hooks/tests/test_log_relation.py	1970-01-01 00:00:00 +0000
+++ hooks/tests/test_log_relation.py	2016-04-26 22:51:55 +0000
@@ -0,0 +1,54 @@
+from testtools import TestCase
+import mock
+import hooks
+hooks.log = mock.MagicMock()
+
+
+class TestLogRelation(TestCase):
+
+    @mock.patch('hooks.config_get')
+    def test_get_log_files_no_raise(self, config_get):
+        'does not raise'
+        hooks.get_log_files()
+
+    @mock.patch('hooks.config_get')
+    def test_get_log_files_parses(self, config_get):
+        'parses files for real'
+        config_get.return_value = {'servername': 'hello'}
+        m = mock.mock_open(read_data='''junk
+            CustomLog /var/log/apache2/access.log combined\n")
+            ErrorLog /var/log/apache2/error.log\n")
+            junk\n"''')
+
+        with mock.patch('hooks.open', m, create=True):
+            access_logs, error_logs = hooks.get_log_files()
+            self.assertEquals('/var/log/apache2/access.log', access_logs[0])
+            self.assertEquals('/var/log/apache2/error.log', error_logs[0])
+
+    @mock.patch('hooks.relation_ids')
+    @mock.patch('hooks.relation_set')
+    def test_log_relation_joined(self, relation_set, relation_ids):
+        relation_ids.return_value = ['logs:1']
+        with mock.patch('hooks.get_log_files') as get_log_files:
+            get_log_files.return_value = ['myaccess_log'], ['myerror_log']
+            hooks.logs_relation_joined()
+            self.assertTrue(relation_set.called)
+            self.assertEquals({'files': 'myaccess_log\nmyerror_log',
+                               'types': 'apache_access\napache_error',
+                               },
+                              relation_set.call_args[1]['relation_settings'])
+
+    @mock.patch('hooks.relation_ids')
+    @mock.patch('hooks.relation_set')
+    def test_log_relation_joined_config_changed(self,
+                                                relation_set,
+                                                relation_ids):
+        relation_ids.return_value = ['logs/0']
+        with mock.patch('hooks.get_log_files') as get_log_files:
+            get_log_files.return_value = ['myaccess_log'], ['myerror_log']
+            hooks.logs_relation_joined()
+            self.assertTrue(relation_set.called)
+            self.assertEquals({'files': 'myaccess_log\nmyerror_log',
+                               'types': 'apache_access\napache_error',
+                               },
+                              relation_set.call_args[1]['relation_settings'])

=== modified file 'hooks/tests/test_vhost_config_relation.py'
--- hooks/tests/test_vhost_config_relation.py	2014-12-29 19:37:05 +0000
+++ hooks/tests/test_vhost_config_relation.py	2016-04-26 22:51:55 +0000
@@ -27,23 +27,19 @@
 
     @patch('hooks.log')
     @patch('subprocess.call')
-    @patch('hooks.close_port')
     def test_create_vhost_missing_template(
-            self, mock_close_port, mock_call, mock_log):
+            self, mock_call, mock_log):
         """Create a vhost file, check contents."""
         hooks.create_vhost(80)
         mock_log.assert_called_once_with(
             "Vhost Template not provided, not configuring: %s" % 80)
-        mock_close_port.assert_called_once()
         self.assertEqual(
             len(os.listdir("%s/%s" % (self.dirname, "sites-available"))), 0)
 
     @patch('hooks.log')
     @patch('subprocess.call')
-    @patch('hooks.close_port')
-    @patch('hooks.open_port')
     def test_create_vhost_template_through_config_no_protocol(
-            self, mock_open_port, mock_close_port, mock_call, mock_log):
+            self, mock_call, mock_log):
         """Create a vhost file, check contents."""
         template = b64encode("http://{{ variable }}/")
         config_data = {
@@ -59,17 +55,13 @@
         with open(filename, 'r') as file:
             contents = file.read()
         self.assertEqual(contents, 'http://fantastic/')
-        mock_open_port.assert_called_once()
-        mock_close_port.assert_called_once()
         self.assertEqual(
             len(os.listdir("%s/%s" % (self.dirname, "sites-available"))), 1)
 
     @patch('hooks.log')
     @patch('subprocess.call')
-    @patch('hooks.close_port')
-    @patch('hooks.open_port')
     def test_create_vhost_template_through_config_with_protocol(
-            self, mock_open_port, mock_close_port, mock_call, mock_log):
+            self, mock_call, mock_log):
         """Create a vhost file, check contents."""
         template = b64encode("http://{{ variable }}/")
         config_data = {
@@ -85,18 +77,13 @@
         with open(filename, 'r') as file:
             contents = file.read()
         self.assertEqual(contents, 'http://fantastic/')
-        mock_open_port.assert_called_once()
-        mock_close_port.assert_called_once()
         self.assertEqual(
             len(os.listdir("%s/%s" % (self.dirname, "sites-available"))), 1)
 
     @patch('hooks.log')
     @patch('subprocess.call')
-    @patch('hooks.close_port')
-    @patch('hooks.open_port')
     def test_create_vhost_template_directly(
-            self, mock_open_port, mock_close_port,
-            mock_call, mock_log):
+            self, mock_call, mock_log):
         """Create a vhost file, check contents."""
         template = b64encode("http://{{ variable }}/")
         config_data = {
@@ -111,8 +98,6 @@
         with open(filename, 'r') as file:
             contents = file.read()
         self.assertEqual(contents, 'http://fantastic/')
-        mock_open_port.assert_called_once()
-        mock_close_port.assert_called_once()
         self.assertEqual(
             len(os.listdir("%s/%s" % (self.dirname, "sites-available"))), 1)
 

=== modified file 'metadata.yaml'
--- metadata.yaml	2014-12-18 01:21:23 +0000
+++ metadata.yaml	2016-04-26 22:51:55 +0000
@@ -18,6 +18,8 @@
     scope: container
   website:
     interface: http
+  logs:
+    interface: logs
   apache-website:
     interface: apache-website
     scope: container

=== modified file 'tests/10-bundles-test.py'
--- tests/10-bundles-test.py	2015-03-10 10:56:19 +0000
+++ tests/10-bundles-test.py	2016-04-26 22:51:55 +0000
@@ -20,8 +20,8 @@
         with open(bundle_path, 'r') as bundle_file:
             contents = yaml.safe_load(bundle_file)
             d.load(contents)
-        d.setup(seconds_to_wait)
-        d.sentry.wait(seconds_to_wait)
+        d.setup(timeout=seconds_to_wait)
+        d.sentry.wait(timeout=seconds_to_wait)
         cls.d = d
 
     def test_deployed(self):

=== modified file 'tests/20-mpm-test.py'
--- tests/20-mpm-test.py	2015-03-10 10:56:19 +0000
+++ tests/20-mpm-test.py	2016-04-26 22:51:55 +0000
@@ -4,6 +4,8 @@
 import unittest
 import amulet
 
+seconds_to_wait = 600
+
 
 class BundleTest(unittest.TestCase):
     """ Create a class for testing the charm in the unit test framework. """
@@ -12,9 +14,10 @@
         """ Set up an amulet deployment using the bundle. """
         d = amulet.Deployment(series='trusty')
         d.add('apache2', os.path.join(os.path.dirname(__file__), os.pardir))
-        d.setup(180)
+        d.setup(timeout=seconds_to_wait)
+        d.sentry.wait(timeout=seconds_to_wait)
         cls.d = d
-        cls.unit = d.sentry.unit['apache2/0']
+        cls.unit = d.sentry['apache2'][0]
         output, code = cls.unit.run('curl localhost')
 
     def assert_mpm(self, mpm):