Merge lp:~fcorrea/charms/trusty/glance/fix-pause-action into lp:~openstack-charmers-archive/charms/trusty/glance/next

[Fernando Correa Neto]

This branch changes the pause action to change the kv database instead of calling set_os_workload_status directly, which is the same pattern used in the swift charm.
This prevents the charm from immediately bouncing back to active after a 'pause' action was performed.

A follow up branch will add a bit more logic to deal with hacluster so it stops sending requests for the unit.

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #14311 glance for fcorrea mp278498
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/14311/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #13339 glance for fcorrea mp278498
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/13339/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #14312 glance-next for fcorrea mp278505
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/14312/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #13340 glance-next for fcorrea mp278505
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/13340/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #8030 glance-next for fcorrea mp278505
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [functional_test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/13495963/
Build: http://10.245.162.77:8080/job/charm_amulet_test/8030/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #8031 glance-next for fcorrea mp278505
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [functional_test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/13500347/
Build: http://10.245.162.77:8080/job/charm_amulet_test/8031/

[Ryan Beisner (1chb1n)]

FYI, undercloud issue caused the last amulet failure. Rerunning...

machine
3 error pending trusty

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #8050 glance-next for fcorrea mp278505
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [functional_test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/13580646/
Build: http://10.245.162.77:8080/job/charm_amulet_test/8050/

[James Page (james-page)]

Same change already found in /next - assuming that someone else got to it first.

Thanks for your work on this!

=== modified file 'actions/actions.py'
--- actions/actions.py	2015-08-26 13:16:31 +0000
+++ actions/actions.py	2015-11-24 20:07:18 +0000
@@ -5,8 +5,9 @@
 
 from charmhelpers.core.host import service_pause, service_resume
 from charmhelpers.core.hookenv import action_fail, status_set
+from charmhelpers.core.unitdata import HookData, kv
 
-from hooks.glance_utils import services
+from hooks.glance_utils import services, assess_status
 
 
 def pause(args):
@@ -18,8 +19,10 @@
         stopped = service_pause(service)
         if not stopped:
             raise Exception("{} didn't stop cleanly.".format(service))
-    status_set(
-        "maintenance", "Paused. Use 'resume' action to resume normal service.")
+    with HookData()():
+        kv().set('unit-paused', True)
+    state, message = assess_status()
+    status_set(state, message)
 
 
 def resume(args):

=== modified file 'charmhelpers/cli/__init__.py'
--- charmhelpers/cli/__init__.py	2015-08-18 17:34:34 +0000
+++ charmhelpers/cli/__init__.py	2015-11-24 20:07:18 +0000
@@ -20,7 +20,7 @@
 
 from six.moves import zip
 
-from charmhelpers.core import unitdata
+import charmhelpers.core.unitdata
 
 
 class OutputFormatter(object):
@@ -163,8 +163,8 @@
         if getattr(arguments.func, '_cli_no_output', False):
             output = ''
         self.formatter.format_output(output, arguments.format)
-        if unitdata._KV:
-            unitdata._KV.flush()
+        if charmhelpers.core.unitdata._KV:
+            charmhelpers.core.unitdata._KV.flush()
 
 
 cmdline = CommandLine()

=== modified file 'charmhelpers/contrib/charmsupport/nrpe.py'
--- charmhelpers/contrib/charmsupport/nrpe.py	2015-04-19 09:00:04 +0000
+++ charmhelpers/contrib/charmsupport/nrpe.py	2015-11-24 20:07:18 +0000
@@ -148,6 +148,13 @@
         self.description = description
         self.check_cmd = self._locate_cmd(check_cmd)
 
+    def _get_check_filename(self):
+        return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command))
+
+    def _get_service_filename(self, hostname):
+        return os.path.join(NRPE.nagios_exportdir,
+                            'service__{}_{}.cfg'.format(hostname, self.command))
+
     def _locate_cmd(self, check_cmd):
         search_path = (
             '/usr/lib/nagios/plugins',
@@ -163,9 +170,21 @@
         log('Check command not found: {}'.format(parts[0]))
         return ''
 
+    def _remove_service_files(self):
+        if not os.path.exists(NRPE.nagios_exportdir):
+            return
+        for f in os.listdir(NRPE.nagios_exportdir):
+            if f.endswith('_{}.cfg'.format(self.command)):
+                os.remove(os.path.join(NRPE.nagios_exportdir, f))
+
+    def remove(self, hostname):
+        nrpe_check_file = self._get_check_filename()
+        if os.path.exists(nrpe_check_file):
+            os.remove(nrpe_check_file)
+        self._remove_service_files()
+
     def write(self, nagios_context, hostname, nagios_servicegroups):
-        nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
-            self.command)
+        nrpe_check_file = self._get_check_filename()
         with open(nrpe_check_file, 'w') as nrpe_check_config:
             nrpe_check_config.write("# check {}\n".format(self.shortname))
             nrpe_check_config.write("command[{}]={}\n".format(
@@ -180,9 +199,7 @@
 
     def write_service_config(self, nagios_context, hostname,
                              nagios_servicegroups):
-        for f in os.listdir(NRPE.nagios_exportdir):
-            if re.search('.*{}.cfg'.format(self.command), f):
-                os.remove(os.path.join(NRPE.nagios_exportdir, f))
+        self._remove_service_files()
 
         templ_vars = {
             'nagios_hostname': hostname,
@@ -192,8 +209,7 @@
             'command': self.command,
         }
         nrpe_service_text = Check.service_template.format(**templ_vars)
-        nrpe_service_file = '{}/service__{}_{}.cfg'.format(
-            NRPE.nagios_exportdir, hostname, self.command)
+        nrpe_service_file = self._get_service_filename(hostname)
         with open(nrpe_service_file, 'w') as nrpe_service_config:
             nrpe_service_config.write(str(nrpe_service_text))
 
@@ -218,12 +234,32 @@
         if hostname:
             self.hostname = hostname
         else:
-            self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
+            nagios_hostname = get_nagios_hostname()
+            if nagios_hostname:
+                self.hostname = nagios_hostname
+            else:
+                self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
         self.checks = []
 
     def add_check(self, *args, **kwargs):
         self.checks.append(Check(*args, **kwargs))
 
+    def remove_check(self, *args, **kwargs):
+        if kwargs.get('shortname') is None:
+            raise ValueError('shortname of check must be specified')
+
+        # Use sensible defaults if they're not specified - these are not
+        # actually used during removal, but they're required for constructing
+        # the Check object; check_disk is chosen because it's part of the
+        # nagios-plugins-basic package.
+        if kwargs.get('check_cmd') is None:
+            kwargs['check_cmd'] = 'check_disk'
+        if kwargs.get('description') is None:
+            kwargs['description'] = ''
+
+        check = Check(*args, **kwargs)
+        check.remove(self.hostname)
+
     def write(self):
         try:
             nagios_uid = pwd.getpwnam('nagios').pw_uid

=== modified file 'charmhelpers/contrib/openstack/amulet/deployment.py'
--- charmhelpers/contrib/openstack/amulet/deployment.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/contrib/openstack/amulet/deployment.py	2015-11-24 20:07:18 +0000
@@ -14,12 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,9 +34,12 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -38,6 +47,22 @@
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
@@ -45,6 +70,8 @@
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
 
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
         # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
@@ -82,6 +109,8 @@
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -95,7 +124,8 @@
                       'ceph-osd', 'ceph-radosgw']
 
         # Charms which can not use openstack-origin, ie. many subordinates
-        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',
+                     'openvswitch-odl', 'neutron-api-odl', 'odl-controller']
 
         if self.openstack:
             for svc in services:
@@ -111,9 +141,79 @@
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 

=== modified file 'charmhelpers/contrib/openstack/amulet/utils.py'
--- charmhelpers/contrib/openstack/amulet/utils.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/contrib/openstack/amulet/utils.py	2015-11-24 20:07:18 +0000
@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
@@ -604,7 +605,22 @@
                            '{}'.format(sample_type, samples))
             return None
 
-# rabbitmq/amqp specific helpers:
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
     def add_rmq_test_user(self, sentry_units,
                           username="testuser1", password="changeme"):
         """Add a test user via the first rmq juju unit, check connection as
@@ -805,7 +821,10 @@
         if port:
             config['ssl_port'] = port
 
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0
@@ -832,7 +851,10 @@
 
         # Disable RMQ SSL
         config = {'ssl': 'off'}
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0

=== modified file 'charmhelpers/contrib/openstack/context.py'
--- charmhelpers/contrib/openstack/context.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/contrib/openstack/context.py	2015-11-24 20:07:18 +0000
@@ -952,6 +952,19 @@
                     'config': config}
         return ovs_ctxt
 
+    def midonet_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        midonet_config = neutron_plugin_attribute(self.plugin, 'config',
+                                                  self.network_manager)
+        mido_ctxt = {'core_plugin': driver,
+                     'neutron_plugin': 'midonet',
+                     'neutron_security_groups': self.neutron_security_groups,
+                     'local_ip': unit_private_ip(),
+                     'config': midonet_config}
+
+        return mido_ctxt
+
     def __call__(self):
         if self.network_manager not in ['quantum', 'neutron']:
             return {}
@@ -973,6 +986,8 @@
             ctxt.update(self.nuage_ctxt())
         elif self.plugin == 'plumgrid':
             ctxt.update(self.pg_ctxt())
+        elif self.plugin == 'midonet':
+            ctxt.update(self.midonet_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')
         if alchemy_flags:
@@ -1073,6 +1088,20 @@
                 config_flags_parser(config_flags)}
 
 
+class LibvirtConfigFlagsContext(OSContextGenerator):
+    """
+    This context provides support for extending
+    the libvirt section through user-defined flags.
+    """
+    def __call__(self):
+        ctxt = {}
+        libvirt_flags = config('libvirt-flags')
+        if libvirt_flags:
+            ctxt['libvirt_flags'] = config_flags_parser(
+                libvirt_flags)
+        return ctxt
+
+
 class SubordinateConfigContext(OSContextGenerator):
 
     """
@@ -1105,7 +1134,7 @@
 
         ctxt = {
             ... other context ...
-            'subordinate_config': {
+            'subordinate_configuration': {
                 'DEFAULT': {
                     'key1': 'value1',
                 },
@@ -1146,22 +1175,23 @@
                     try:
                         sub_config = json.loads(sub_config)
                     except:
-                        log('Could not parse JSON from subordinate_config '
-                            'setting from %s' % rid, level=ERROR)
+                        log('Could not parse JSON from '
+                            'subordinate_configuration setting from %s'
+                            % rid, level=ERROR)
                         continue
 
                     for service in self.services:
                         if service not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
-                                'nothing for %s service' % (rid, service),
-                                level=INFO)
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s service'
+                                % (rid, service), level=INFO)
                             continue
 
                         sub_config = sub_config[service]
                         if self.config_file not in sub_config:
-                            log('Found subordinate_config on %s but it contained'
-                                'nothing for %s' % (rid, self.config_file),
-                                level=INFO)
+                            log('Found subordinate_configuration on %s but it '
+                                'contained nothing for %s'
+                                % (rid, self.config_file), level=INFO)
                             continue
 
                         sub_config = sub_config[self.config_file]

=== modified file 'charmhelpers/contrib/openstack/neutron.py'
--- charmhelpers/contrib/openstack/neutron.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/contrib/openstack/neutron.py	2015-11-24 20:07:18 +0000
@@ -204,11 +204,25 @@
                                         database=config('database'),
                                         ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
-            'packages': [['plumgrid-lxc'],
-                         ['iovisor-dkms']],
+            'packages': ['plumgrid-lxc',
+                         'iovisor-dkms'],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-plumgrid'],
             'server_services': ['neutron-server']
+        },
+        'midonet': {
+            'config': '/etc/neutron/plugins/midonet/midonet.ini',
+            'driver': 'midonet.neutron.plugin.MidonetPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [[headers_package()] + determine_dkms_package()],
+            'server_packages': ['neutron-server',
+                                'python-neutron-plugin-midonet'],
+            'server_services': ['neutron-server']
         }
     }
     if release >= 'icehouse':

=== modified file 'charmhelpers/contrib/openstack/utils.py'
--- charmhelpers/contrib/openstack/utils.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/contrib/openstack/utils.py	2015-11-24 20:07:18 +0000
@@ -26,6 +26,7 @@
 
 import six
 import traceback
+import uuid
 import yaml
 
 from charmhelpers.contrib.network import ip
@@ -41,6 +42,7 @@
     log as juju_log,
     charm_dir,
     INFO,
+    related_units,
     relation_ids,
     relation_set,
     status_set,
@@ -121,6 +123,7 @@
     ('2.2.2', 'kilo'),
     ('2.3.0', 'liberty'),
     ('2.4.0', 'liberty'),
+    ('2.5.0', 'liberty'),
 ])
 
 # >= Liberty version->codename mapping
@@ -858,7 +861,9 @@
         if charm_state != 'active' and charm_state != 'unknown':
             state = workload_state_compare(state, charm_state)
             if message:
-                message = "{} {}".format(message, charm_message)
+                charm_message = charm_message.replace("Incomplete relations: ",
+                                                      "")
+                message = "{}, {}".format(message, charm_message)
             else:
                 message = charm_message
 
@@ -975,3 +980,19 @@
             action_set({'outcome': 'no upgrade available.'})
 
     return ret
+
+
+def remote_restart(rel_name, remote_service=None):
+    trigger = {
+        'restart-trigger': str(uuid.uuid4()),
+    }
+    if remote_service:
+        trigger['remote-service'] = remote_service
+    for rid in relation_ids(rel_name):
+        # This subordinate can be related to two seperate services using
+        # different subordinate relations so only issue the restart if
+        # the principle is conencted down the relation we think it is
+        if related_units(relid=rid):
+            relation_set(relation_id=rid,
+                         relation_settings=trigger,
+                         )

=== modified file 'charmhelpers/contrib/storage/linux/ceph.py'
--- charmhelpers/contrib/storage/linux/ceph.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/contrib/storage/linux/ceph.py	2015-11-24 20:07:18 +0000
@@ -26,6 +26,7 @@
 
 import os
 import shutil
+import six
 import json
 import time
 import uuid
@@ -125,29 +126,37 @@
     return None
 
 
-def create_pool(service, name, replicas=3):
+def update_pool(client, pool, settings):
+    cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool]
+    for k, v in six.iteritems(settings):
+        cmd.append(k)
+        cmd.append(v)
+
+    check_call(cmd)
+
+
+def create_pool(service, name, replicas=3, pg_num=None):
     """Create a new RADOS pool."""
     if pool_exists(service, name):
         log("Ceph pool {} already exists, skipping creation".format(name),
             level=WARNING)
         return
 
-    # Calculate the number of placement groups based
-    # on upstream recommended best practices.
-    osds = get_osds(service)
-    if osds:
-        pgnum = (len(osds) * 100 // replicas)
-    else:
-        # NOTE(james-page): Default to 200 for older ceph versions
-        # which don't support OSD query from cli
-        pgnum = 200
-
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)]
-    check_call(cmd)
-
-    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size',
-           str(replicas)]
-    check_call(cmd)
+    if not pg_num:
+        # Calculate the number of placement groups based
+        # on upstream recommended best practices.
+        osds = get_osds(service)
+        if osds:
+            pg_num = (len(osds) * 100 // replicas)
+        else:
+            # NOTE(james-page): Default to 200 for older ceph versions
+            # which don't support OSD query from cli
+            pg_num = 200
+
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)]
+    check_call(cmd)
+
+    update_pool(service, name, settings={'size': str(replicas)})
 
 
 def delete_pool(service, name):
@@ -202,10 +211,10 @@
     log('Created new keyfile at %s.' % keyfile, level=INFO)
 
 
-def get_ceph_nodes():
-    """Query named relation 'ceph' to determine current nodes."""
+def get_ceph_nodes(relation='ceph'):
+    """Query named relation to determine current nodes."""
     hosts = []
-    for r_id in relation_ids('ceph'):
+    for r_id in relation_ids(relation):
         for unit in related_units(r_id):
             hosts.append(relation_get('private-address', unit=unit, rid=r_id))
 
@@ -357,14 +366,14 @@
             service_start(svc)
 
 
-def ensure_ceph_keyring(service, user=None, group=None):
+def ensure_ceph_keyring(service, user=None, group=None, relation='ceph'):
     """Ensures a ceph keyring is created for a named service and optionally
     ensures user and group ownership.
 
     Returns False if no ceph key is available in relation state.
     """
     key = None
-    for rid in relation_ids('ceph'):
+    for rid in relation_ids(relation):
         for unit in related_units(rid):
             key = relation_get('key', rid=rid, unit=unit)
             if key:
@@ -413,9 +422,16 @@
             self.request_id = str(uuid.uuid1())
         self.ops = []
 
-    def add_op_create_pool(self, name, replica_count=3):
+    def add_op_create_pool(self, name, replica_count=3, pg_num=None):
+        """Adds an operation to create a pool.
+
+        @param pg_num setting:  optional setting. If not provided, this value
+        will be calculated by the broker based on how many OSDs are in the
+        cluster at the time of creation. Note that, if provided, this value
+        will be capped at the current available maximum.
+        """
         self.ops.append({'op': 'create-pool', 'name': name,
-                         'replicas': replica_count})
+                         'replicas': replica_count, 'pg_num': pg_num})
 
     def set_ops(self, ops):
         """Set request ops to provided value.
@@ -433,8 +449,8 @@
     def _ops_equal(self, other):
         if len(self.ops) == len(other.ops):
             for req_no in range(0, len(self.ops)):
-                for key in ['replicas', 'name', 'op']:
-                    if self.ops[req_no][key] != other.ops[req_no][key]:
+                for key in ['replicas', 'name', 'op', 'pg_num']:
+                    if self.ops[req_no].get(key) != other.ops[req_no].get(key):
                         return False
         else:
             return False
@@ -540,7 +556,7 @@
     return request
 
 
-def get_request_states(request):
+def get_request_states(request, relation='ceph'):
     """Return a dict of requests per relation id with their corresponding
        completion state.
 
@@ -552,7 +568,7 @@
     """
     complete = []
     requests = {}
-    for rid in relation_ids('ceph'):
+    for rid in relation_ids(relation):
         complete = False
         previous_request = get_previous_request(rid)
         if request == previous_request:
@@ -570,14 +586,14 @@
     return requests
 
 
-def is_request_sent(request):
+def is_request_sent(request, relation='ceph'):
     """Check to see if a functionally equivalent request has already been sent
 
     Returns True if a similair request has been sent
 
     @param request: A CephBrokerRq object
     """
-    states = get_request_states(request)
+    states = get_request_states(request, relation=relation)
     for rid in states.keys():
         if not states[rid]['sent']:
             return False
@@ -585,7 +601,7 @@
     return True
 
 
-def is_request_complete(request):
+def is_request_complete(request, relation='ceph'):
     """Check to see if a functionally equivalent request has already been
     completed
 
@@ -593,7 +609,7 @@
 
     @param request: A CephBrokerRq object
     """
-    states = get_request_states(request)
+    states = get_request_states(request, relation=relation)
     for rid in states.keys():
         if not states[rid]['complete']:
             return False
@@ -643,15 +659,15 @@
     return 'broker-rsp-' + local_unit().replace('/', '-')
 
 
-def send_request_if_needed(request):
+def send_request_if_needed(request, relation='ceph'):
     """Send broker request if an equivalent request has not already been sent
 
     @param request: A CephBrokerRq object
     """
-    if is_request_sent(request):
+    if is_request_sent(request, relation=relation):
         log('Request already sent but not complete, not sending new request',
             level=DEBUG)
     else:
-        for rid in relation_ids('ceph'):
+        for rid in relation_ids(relation):
             log('Sending request {}'.format(request.request_id), level=DEBUG)
             relation_set(relation_id=rid, broker_req=request.request)

=== modified file 'charmhelpers/core/hookenv.py'
--- charmhelpers/core/hookenv.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/core/hookenv.py	2015-11-24 20:07:18 +0000
@@ -491,6 +491,19 @@
 
 
 @cached
+def peer_relation_id():
+    '''Get a peer relation id if a peer relation has been joined, else None.'''
+    md = metadata()
+    section = md.get('peers')
+    if section:
+        for key in section:
+            relids = relation_ids(key)
+            if relids:
+                return relids[0]
+    return None
+
+
+@cached
 def relation_to_interface(relation_name):
     """
     Given the name of a relation, return the interface that relation uses.
@@ -624,7 +637,7 @@
 
 
 @cached
-def storage_get(attribute="", storage_id=""):
+def storage_get(attribute=None, storage_id=None):
     """Get storage attributes"""
     _args = ['storage-get', '--format=json']
     if storage_id:
@@ -638,7 +651,7 @@
 
 
 @cached
-def storage_list(storage_name=""):
+def storage_list(storage_name=None):
     """List the storage IDs for the unit"""
     _args = ['storage-list', '--format=json']
     if storage_name:
@@ -820,6 +833,7 @@
 
 def translate_exc(from_exc, to_exc):
     def inner_translate_exc1(f):
+        @wraps(f)
         def inner_translate_exc2(*args, **kwargs):
             try:
                 return f(*args, **kwargs)

=== modified file 'charmhelpers/core/host.py'
--- charmhelpers/core/host.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/core/host.py	2015-11-24 20:07:18 +0000
@@ -67,7 +67,9 @@
     """Pause a system service.
 
     Stop it, and prevent it from starting again at boot."""
-    stopped = service_stop(service_name)
+    stopped = True
+    if service_running(service_name):
+        stopped = service_stop(service_name)
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
     if os.path.exists(upstart_file):
@@ -105,7 +107,9 @@
             "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
                 service_name, upstart_file, sysv_file))
 
-    started = service_start(service_name)
+    started = service_running(service_name)
+    if not started:
+        started = service_start(service_name)
     return started
 
 
@@ -566,7 +570,14 @@
         os.chdir(cur)
 
 
-def chownr(path, owner, group, follow_links=True):
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
+    """
+    Recursively change user and group ownership of files and directories
+    in given path. Doesn't chown path itself by default, only its children.
+
+    :param bool follow_links: Also Chown links if True
+    :param bool chowntopdir: Also chown path itself if True
+    """
     uid = pwd.getpwnam(owner).pw_uid
     gid = grp.getgrnam(group).gr_gid
     if follow_links:
@@ -574,6 +585,10 @@
     else:
         chown = os.lchown
 
+    if chowntopdir:
+        broken_symlink = os.path.lexists(path) and not os.path.exists(path)
+        if not broken_symlink:
+            chown(path, uid, gid)
     for root, dirs, files in os.walk(path):
         for name in dirs + files:
             full = os.path.join(root, name)
@@ -584,3 +599,19 @@
 
 def lchownr(path, owner, group):
     chownr(path, owner, group, follow_links=False)
+
+
+def get_total_ram():
+    '''The total amount of system RAM in bytes.
+
+    This is what is reported by the OS, and may be overcommitted when
+    there are multiple containers hosted on the same machine.
+    '''
+    with open('/proc/meminfo', 'r') as f:
+        for line in f.readlines():
+            if line:
+                key, value, unit = line.split()
+                if key == 'MemTotal:':
+                    assert unit == 'kB', 'Unknown unit'
+                    return int(value) * 1024  # Classic, not KiB.
+        raise NotImplementedError()

=== modified file 'charmhelpers/core/hugepage.py'
--- charmhelpers/core/hugepage.py	2015-09-30 15:01:18 +0000
+++ charmhelpers/core/hugepage.py	2015-11-24 20:07:18 +0000
@@ -46,6 +46,8 @@
     group_info = add_group(group)
     gid = group_info.gr_gid
     add_user_to_group(user, group)
+    if max_map_count < 2 * nr_hugepages:
+        max_map_count = 2 * nr_hugepages
     sysctl_settings = {
         'vm.nr_hugepages': nr_hugepages,
         'vm.max_map_count': max_map_count,

=== modified file 'charmhelpers/core/services/helpers.py'
--- charmhelpers/core/services/helpers.py	2015-08-18 17:34:34 +0000
+++ charmhelpers/core/services/helpers.py	2015-11-24 20:07:18 +0000
@@ -249,16 +249,18 @@
     :param int perms: The permissions of the rendered file
     :param partial on_change_action: functools partial to be executed when
                                      rendered file changes
+    :param jinja2 loader template_loader: A jinja2 template loader
     """
     def __init__(self, source, target,
                  owner='root', group='root', perms=0o444,
-                 on_change_action=None):
+                 on_change_action=None, template_loader=None):
         self.source = source
         self.target = target
         self.owner = owner
         self.group = group
         self.perms = perms
         self.on_change_action = on_change_action
+        self.template_loader = template_loader
 
     def __call__(self, manager, service_name, event_name):
         pre_checksum = ''
@@ -269,7 +271,8 @@
         for ctx in service.get('required_data', []):
             context.update(ctx)
         templating.render(self.source, self.target, context,
-                          self.owner, self.group, self.perms)
+                          self.owner, self.group, self.perms,
+                          template_loader=self.template_loader)
         if self.on_change_action:
             if pre_checksum == host.file_hash(self.target):
                 hookenv.log(

=== modified file 'charmhelpers/core/templating.py'
--- charmhelpers/core/templating.py	2015-03-20 17:15:02 +0000
+++ charmhelpers/core/templating.py	2015-11-24 20:07:18 +0000
@@ -21,7 +21,7 @@
 
 
 def render(source, target, context, owner='root', group='root',
-           perms=0o444, templates_dir=None, encoding='UTF-8'):
+           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
     """
     Render a template.
 
@@ -52,17 +52,24 @@
         apt_install('python-jinja2', fatal=True)
         from jinja2 import FileSystemLoader, Environment, exceptions
 
-    if templates_dir is None:
-        templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
-    loader = Environment(loader=FileSystemLoader(templates_dir))
+    if template_loader:
+        template_env = Environment(loader=template_loader)
+    else:
+        if templates_dir is None:
+            templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+        template_env = Environment(loader=FileSystemLoader(templates_dir))
     try:
         source = source
-        template = loader.get_template(source)
+        template = template_env.get_template(source)
     except exceptions.TemplateNotFound as e:
         hookenv.log('Could not load template %s from %s.' %
                     (source, templates_dir),
                     level=hookenv.ERROR)
         raise e
     content = template.render(context)
-    host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
+    target_dir = os.path.dirname(target)
+    if not os.path.exists(target_dir):
+        # This is a terrible default directory permission, as the file
+        # or its siblings will often contain secrets.
+        host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
     host.write_file(target, content.encode(encoding), owner, group, perms)

=== modified file 'charmhelpers/fetch/__init__.py'
--- charmhelpers/fetch/__init__.py	2015-08-18 17:34:34 +0000
+++ charmhelpers/fetch/__init__.py	2015-11-24 20:07:18 +0000
@@ -225,12 +225,12 @@
 
 def apt_mark(packages, mark, fatal=False):
     """Flag one or more packages using apt-mark"""
+    log("Marking {} as {}".format(packages, mark))
     cmd = ['apt-mark', mark]
     if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
-    log("Holding {}".format(packages))
 
     if fatal:
         subprocess.check_call(cmd, universal_newlines=True)

=== modified file 'charmhelpers/fetch/bzrurl.py'
--- charmhelpers/fetch/bzrurl.py	2015-03-20 17:15:02 +0000
+++ charmhelpers/fetch/bzrurl.py	2015-11-24 20:07:18 +0000
@@ -64,11 +64,15 @@
         except Exception as e:
             raise e
 
-    def install(self, source):
+    def install(self, source, dest=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
-        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
-                                branch_name)
+        if dest:
+            dest_dir = os.path.join(dest, branch_name)
+        else:
+            dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+                                    branch_name)
+
         if not os.path.exists(dest_dir):
             mkdir(dest_dir, perms=0o755)
         try:

=== modified file 'hooks/glance_utils.py'
--- hooks/glance_utils.py	2015-10-30 22:11:38 +0000
+++ hooks/glance_utils.py	2015-11-24 20:07:18 +0000
@@ -70,6 +70,8 @@
     retry_on_exception,
 )
 
+from charmhelpers.core.unitdata import HookData, kv
+
 
 CLUSTER_RES = "grp_glance_vips"
 
@@ -491,3 +493,21 @@
     swift_connection.post_account(headers={'x-account-meta-temp-url-key':
                                            temp_url_key})
     return temp_url_key
+
+
+def assess_status():
+    """Assess status of current unit"""
+    if is_paused():
+        return ("maintenance",
+                "Paused. Use 'resume' action to resume normal service.")
+    else:
+        return ("active", "Unit is ready")
+
+
+def is_paused():
+    """Is the unit paused?"""
+    with HookData()():
+        if kv().get('unit-paused'):
+            return True
+        else:
+            return False

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-09-30 15:01:18 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2015-11-24 20:07:18 +0000
@@ -14,12 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -28,9 +34,12 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -38,6 +47,22 @@
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
@@ -45,6 +70,8 @@
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
 
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
         # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
@@ -82,6 +109,8 @@
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -95,7 +124,8 @@
                       'ceph-osd', 'ceph-radosgw']
 
         # Charms which can not use openstack-origin, ie. many subordinates
-        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
+        no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',
+                     'openvswitch-odl', 'neutron-api-odl', 'odl-controller']
 
         if self.openstack:
             for svc in services:
@@ -111,9 +141,79 @@
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
+    def _auto_wait_for_status(self, message=None, exclude_services=None,
+                              include_only=None, timeout=1800):
+        """Wait for all units to have a specific extended status, except
+        for any defined as excluded.  Unless specified via message, any
+        status containing any case of 'ready' will be considered a match.
+
+        Examples of message usage:
+
+          Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
+              message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
+
+          Wait for all units to reach this status (exact match):
+              message = re.compile('^Unit is ready and clustered$')
+
+          Wait for all units to reach any one of these (exact match):
+              message = re.compile('Unit is ready|OK|Ready')
+
+          Wait for at least one unit to reach this status (exact match):
+              message = {'ready'}
+
+        See Amulet's sentry.wait_for_messages() for message usage detail.
+        https://github.com/juju/amulet/blob/master/amulet/sentry.py
+
+        :param message: Expected status match
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
+        :param timeout: Maximum time in seconds to wait for status match
+        :returns: None.  Raises if timeout is hit.
+        """
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
+            message = re.compile('.*ready.*', re.IGNORECASE)
+
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
+            exclude_services = []
+
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
+        service_messages = {service: message for service in services}
+        self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
+
     def _get_openstack_release(self):
         """Get openstack release.
 

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-09-30 15:01:18 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2015-11-24 20:07:18 +0000
@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
@@ -604,7 +605,22 @@
                            '{}'.format(sample_type, samples))
             return None
 
-# rabbitmq/amqp specific helpers:
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
     def add_rmq_test_user(self, sentry_units,
                           username="testuser1", password="changeme"):
         """Add a test user via the first rmq juju unit, check connection as
@@ -805,7 +821,10 @@
         if port:
             config['ssl_port'] = port
 
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0
@@ -832,7 +851,10 @@
 
         # Disable RMQ SSL
         config = {'ssl': 'off'}
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0

=== modified file 'unit_tests/test_actions.py'
--- unit_tests/test_actions.py	2015-09-02 11:40:06 +0000
+++ unit_tests/test_actions.py	2015-11-24 20:07:18 +0000
@@ -12,7 +12,8 @@
 
     def setUp(self):
         super(PauseTestCase, self).setUp(
-            actions.actions, ["service_pause", "status_set"])
+            actions.actions, ["service_pause", "status_set",
+                              "HookData", "kv", "assess_status"])
 
     def test_pauses_services(self):
         """Pause action pauses all Glance services."""
@@ -23,6 +24,7 @@
             return True
 
         self.service_pause.side_effect = fake_service_pause
+        self.assess_status.return_value = ("maintenance", "Foo",)
 
         actions.actions.pause([])
         self.assertItemsEqual(
@@ -48,18 +50,21 @@
 
     def test_status_mode(self):
         """Pause action sets the status to maintenance."""
-        status_calls = []
-        self.status_set.side_effect = lambda state, msg: status_calls.append(
-            state)
+        self.HookData()().return_value = True
+        self.assess_status.return_value = ("maintenance", "Foo",)
 
         actions.actions.pause([])
-        self.assertEqual(status_calls, ["maintenance"])
+        self.kv().set.assert_called_with('unit-paused', True)
 
     def test_status_message(self):
         """Pause action sets a status message reflecting that it's paused."""
         status_calls = []
         self.status_set.side_effect = lambda state, msg: status_calls.append(
             msg)
+        self.HookData()().return_value = True
+        self.assess_status.return_value = (
+            "maintenance", "Paused. Use 'resume' action to resume normal"
+            " service.")
 
         actions.actions.pause([])
         self.assertEqual(