Merge lp:~corey.bryant/charm-helpers/wsgi into lp:charm-helpers

[Corey Bryant]

Add WSGI context and template for OpenStack APIs

[lp:~corey.bryant/charm-helpers/wsgi]

667. By Liam Young on 2016-12-12

[gnuoy, r=ajkavanagh] Fixes for amulet memcache service checks.
1) Use 'service' not 'status' for checking status of memcached on pre-systemd systems as its not an upstart script.
2) Trusty memcache uses ip6-localhost as its listen address because it does not understand ::1

668. By Stuart Bishop on 2016-12-13

[hloeung,r=stub] Add timeout to rsync and capture rsync error messages

669. By Liam Young on 2016-12-14

[gnuoy, r=ajkavanagh] Add option package var to Memcache context to allow contect to lookup Openstack release without using Openstack-origin which may not be present. enable_memcache also had to be extended to take a package option to support this

[Liam Young (gnuoy)]

It looks good but needs unit tests for the new context and methods

review: Needs Fixing

[lp:~corey.bryant/charm-helpers/wsgi]

670. By Corey Bryant on 2016-12-14

Add WSGI context and template for OpenStack APIs

671. By Corey Bryant on 2016-12-14

Add unit test for WSGI context.

672. By Corey Bryant on 2016-12-14

Add unit test for git_determine_*()

[Corey Bryant (corey.bryant)]

Thanks for the review Liam. I've added unit tests.

[Liam Young (gnuoy)]

LGTM, thanks

review: Approve

=== modified file 'charmhelpers/contrib/openstack/context.py'
--- charmhelpers/contrib/openstack/context.py	2016-12-14 15:10:32 +0000
+++ charmhelpers/contrib/openstack/context.py	2016-12-14 18:51:33 +0000
@@ -14,6 +14,7 @@
 
 import glob
 import json
+import math
 import os
 import re
 import time
@@ -90,6 +91,8 @@
 from charmhelpers.contrib.openstack.utils import (
     config_flags_parser,
     get_host_ip,
+    git_determine_usr_bin,
+    git_determine_python_path,
     enable_memcache,
 )
 from charmhelpers.core.unitdata import kv
@@ -1208,6 +1211,43 @@
         return ctxt
 
 
+class WSGIWorkerConfigContext(WorkerConfigContext):
+
+    def __init__(self, name=None, script=None, admin_script=None,
+                 public_script=None, process_weight=1.00,
+                 admin_process_weight=0.75, public_process_weight=0.25):
+        self.service_name = name
+        self.user = name
+        self.group = name
+        self.script = script
+        self.admin_script = admin_script
+        self.public_script = public_script
+        self.process_weight = process_weight
+        self.admin_process_weight = admin_process_weight
+        self.public_process_weight = public_process_weight
+
+    def __call__(self):
+        multiplier = config('worker-multiplier') or 1
+        total_processes = self.num_cpus * multiplier
+        ctxt = {
+            "service_name": self.service_name,
+            "user": self.user,
+            "group": self.group,
+            "script": self.script,
+            "admin_script": self.admin_script,
+            "public_script": self.public_script,
+            "processes": int(math.ceil(self.process_weight * total_processes)),
+            "admin_processes": int(math.ceil(self.admin_process_weight *
+                                             total_processes)),
+            "public_processes": int(math.ceil(self.public_process_weight *
+                                              total_processes)),
+            "threads": 1,
+            "usr_bin": git_determine_usr_bin(),
+            "python_path": git_determine_python_path(),
+        }
+        return ctxt
+
+
 class ZeroMQContext(OSContextGenerator):
     interfaces = ['zeromq-configuration']
 

=== added file 'charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf'
--- charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf	1970-01-01 00:00:00 +0000
+++ charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf	2016-12-14 18:51:33 +0000
@@ -0,0 +1,100 @@
+# Configuration file maintained by Juju. Local changes may be overwritten.
+
+{% if port -%}
+Listen {{ port }}
+{% endif -%}
+
+{% if admin_port -%}
+Listen {{ admin_port }}
+{% endif -%}
+
+{% if public_port -%}
+Listen {{ public_port }}
+{% endif -%}
+
+{% if port -%}
+<VirtualHost *:{{ port }}>
+    WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}
+    WSGIScriptAlias / {{ script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}
+
+{% if admin_port -%}
+<VirtualHost *:{{ admin_port }}>
+    WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}-admin
+    WSGIScriptAlias / {{ admin_script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}
+
+{% if public_port -%}
+<VirtualHost *:{{ public_port }}>
+    WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
+{% if python_path -%}
+                      python-path={{ python_path }} \
+{% endif -%}
+                      display-name=%{GROUP}
+    WSGIProcessGroup {{ service_name }}-public
+    WSGIScriptAlias / {{ public_script }}
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIPassAuthorization On
+    <IfVersion >= 2.4>
+      ErrorLogFormat "%{cu}t %M"
+    </IfVersion>
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
+
+    <Directory {{ usr_bin }}>
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+        <IfVersion < 2.4>
+            Order allow,deny
+            Allow from all
+        </IfVersion>
+    </Directory>
+</VirtualHost>
+{% endif -%}

=== modified file 'charmhelpers/contrib/openstack/utils.py'
--- charmhelpers/contrib/openstack/utils.py	2016-12-14 15:46:41 +0000
+++ charmhelpers/contrib/openstack/utils.py	2016-12-14 18:51:33 +0000
@@ -1119,6 +1119,35 @@
                 shutil.copyfile(service_source, service_dest)
 
 
+def git_determine_usr_bin():
+    """Return the /usr/bin path for Apache2 config.
+
+    The /usr/bin path will be located in the virtualenv if the charm
+    is configured to deploy from source.
+    """
+    if git_install_requested():
+        projects_yaml = config('openstack-origin-git')
+        projects_yaml = git_default_repos(projects_yaml)
+        return os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
+    else:
+        return '/usr/bin'
+
+
+def git_determine_python_path():
+    """Return the python-path for Apache2 config.
+
+    Returns 'None' unless the charm is configured to deploy from source,
+    in which case the path of the virtualenv's site-packages is returned.
+    """
+    if git_install_requested():
+        projects_yaml = config('openstack-origin-git')
+        projects_yaml = git_default_repos(projects_yaml)
+        return os.path.join(git_pip_venv_dir(projects_yaml),
+                            'lib/python2.7/site-packages')
+    else:
+        return None
+
+
 def os_workload_status(configs, required_interfaces, charm_func=None):
     """
     Decorator to set workload status based on complete contexts

=== modified file 'tests/contrib/openstack/test_openstack_utils.py'
--- tests/contrib/openstack/test_openstack_utils.py	2016-09-16 09:50:01 +0000
+++ tests/contrib/openstack/test_openstack_utils.py	2016-12-14 18:51:33 +0000
@@ -933,6 +933,40 @@
         openstack.git_src_dir(openstack_origin_git, 'keystone')
         join.assert_called_with('/mnt/openstack-git', 'keystone')
 
+    @patch.object(openstack, 'config')
+    def test_git_determine_usr_bin(self, config):
+        config.return_value = None
+        result = openstack.git_determine_usr_bin()
+        self.assertEquals(result, '/usr/bin')
+
+    @patch('os.path.join')
+    @patch.object(openstack, 'git_default_repos')
+    @patch.object(openstack, 'config')
+    def test_git_determine_usr_bin_git(self, config, git_default, join):
+        venv_bin = '/mnt/openstack/.venv/bin'
+        join.return_value = venv_bin
+        config.return_value = openstack_origin_git
+        git_default.return_value = openstack_origin_git
+        result = openstack.git_determine_usr_bin()
+        self.assertEquals(result, venv_bin)
+
+    @patch.object(openstack, 'config')
+    def test_git_determine_python_path(self, config):
+        config.return_value = None
+        result = openstack.git_determine_python_path()
+        self.assertEquals(result, None)
+
+    @patch('os.path.join')
+    @patch.object(openstack, 'git_default_repos')
+    @patch.object(openstack, 'config')
+    def test_git_determine_python_path_git(self, config, git_default, join):
+        venv_site = '/mnt/openstack/.venv/lib/python2.7/site-packages'
+        join.return_value = venv_site
+        config.return_value = openstack_origin_git
+        git_default.return_value = openstack_origin_git
+        result = openstack.git_determine_python_path()
+        self.assertEquals(result, venv_site)
+
     def test_incomplete_relation_data(self):
         configs = MagicMock()
         configs.complete_contexts.return_value = ['pgsql-db', 'amqp']

=== modified file 'tests/contrib/openstack/test_os_contexts.py'
--- tests/contrib/openstack/test_os_contexts.py	2016-12-09 16:48:53 +0000
+++ tests/contrib/openstack/test_os_contexts.py	2016-12-14 18:51:33 +0000
@@ -2371,6 +2371,35 @@
         }
         self.assertEquals(result, expected)
 
+    @patch.object(context, 'psutil')
+    @patch.object(context, 'git_determine_python_path')
+    @patch.object(context, 'git_determine_usr_bin')
+    def test_wsgi_worker_config_context(self, usr_bin, python_path, psutil):
+        self.config.return_value = 2  # worker-multiplier=2
+        usr_bin_path = '/usr/bin'
+        usr_bin.return_value = usr_bin_path
+        python_path.return_value = None
+        psutil.cpu_count.return_value = 4
+        service_name = 'service-name'
+        script = '/usr/bin/script'
+        ctxt = context.WSGIWorkerConfigContext(name=service_name,
+                                               script=script)
+        expect = {
+            "service_name": service_name,
+            "user": service_name,
+            "group": service_name,
+            "script": script,
+            "admin_script": None,
+            "public_script": None,
+            "processes": 8,
+            "admin_processes": 6,
+            "public_processes": 2,
+            "threads": 1,
+            "usr_bin": usr_bin_path,
+            "python_path": None,
+        }
+        self.assertEqual(expect, ctxt())
+
     def test_zeromq_context_unrelated(self):
         self.is_relation_made.return_value = False
         self.assertEquals(context.ZeroMQContext()(), {})