Merge lp:~1chb1n/charm-helpers/openstack-amulet-ext-stat-wait-update into lp:charm-helpers

[Ryan Beisner]

Add logger to openstack/amulet/deployment.py.

Update _auto_wait_for_status with logging feedback and optional include_only logic.

Update rmq test helpers to use extended status message for a readiness indicator.

[Ryan Beisner (1chb1n)]

Lint check ok: http://paste.ubuntu.com/12883488/

[Liam Young (gnuoy)]

Approve

review: Approve

=== modified file 'charmhelpers/contrib/openstack/amulet/deployment.py'
--- charmhelpers/contrib/openstack/amulet/deployment.py	2015-10-15 20:34:50 +0000
+++ charmhelpers/contrib/openstack/amulet/deployment.py	2015-10-21 03:27:04 +0000
@@ -14,13 +14,18 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import re
+import sys
 import six
 from collections import OrderedDict
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
 
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
 
 class OpenStackAmuletDeployment(AmuletDeployment):
     """OpenStack amulet deployment.
@@ -29,9 +34,12 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None, stable=True):
+    def __init__(self, series=None, openstack=None, source=None,
+                 stable=True, log_level=DEBUG):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
+        self.log = self.get_logger(level=log_level)
+        self.log.info('OpenStackAmuletDeployment:  init')
         self.openstack = openstack
         self.source = source
         self.stable = stable
@@ -39,6 +47,22 @@
         # out.
         self.current_next = "trusty"
 
+    def get_logger(self, name="deployment-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
@@ -46,6 +70,8 @@
            stable or next (dev) branch, and based on this, use the corresonding
            stable or next branches for the other_services."""
 
+        self.log.info('OpenStackAmuletDeployment:  determine branch locations')
+
         # Charms outside the lp:~openstack-charmers namespace
         base_charms = ['mysql', 'mongodb', 'nrpe']
 
@@ -83,6 +109,8 @@
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
+        self.log.info('OpenStackAmuletDeployment:  adding services')
+
         other_services = self._determine_branch_locations(other_services)
 
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
@@ -112,11 +140,12 @@
 
     def _configure_services(self, configs):
         """Configure all of the services."""
+        self.log.info('OpenStackAmuletDeployment:  configure services')
         for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
     def _auto_wait_for_status(self, message=None, exclude_services=None,
-                              timeout=1800):
+                              include_only=None, timeout=1800):
         """Wait for all units to have a specific extended status, except
         for any defined as excluded.  Unless specified via message, any
         status containing any case of 'ready' will be considered a match.
@@ -127,7 +156,7 @@
               message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
 
           Wait for all units to reach this status (exact match):
-              message = 'Unit is ready'
+              message = re.compile('^Unit is ready and clustered$')
 
           Wait for all units to reach any one of these (exact match):
               message = re.compile('Unit is ready|OK|Ready')
@@ -139,20 +168,50 @@
         https://github.com/juju/amulet/blob/master/amulet/sentry.py
 
         :param message: Expected status match
-        :param exclude_services: List of juju service names to ignore
+        :param exclude_services: List of juju service names to ignore,
+            not to be used in conjuction with include_only.
+        :param include_only: List of juju service names to exclusively check,
+            not to be used in conjuction with exclude_services.
         :param timeout: Maximum time in seconds to wait for status match
         :returns: None.  Raises if timeout is hit.
         """
-
-        if not message:
+        self.log.info('Waiting for extended status on units...')
+
+        all_services = self.d.services.keys()
+
+        if exclude_services and include_only:
+            raise ValueError('exclude_services can not be used '
+                             'with include_only')
+
+        if message:
+            if isinstance(message, re._pattern_type):
+                match = message.pattern
+            else:
+                match = message
+
+            self.log.debug('Custom extended status wait match: '
+                           '{}'.format(match))
+        else:
+            self.log.debug('Default extended status wait match:  contains '
+                           'READY (case-insensitive)')
             message = re.compile('.*ready.*', re.IGNORECASE)
 
-        if not exclude_services:
+        if exclude_services:
+            self.log.debug('Excluding services from extended status match: '
+                           '{}'.format(exclude_services))
+        else:
             exclude_services = []
 
-        services = list(set(self.d.services.keys()) - set(exclude_services))
+        if include_only:
+            services = include_only
+        else:
+            services = list(set(all_services) - set(exclude_services))
+
+        self.log.debug('Waiting up to {}s for extended status on services: '
+                       '{}'.format(timeout, services))
         service_messages = {service: message for service in services}
         self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
+        self.log.info('OK')
 
     def _get_openstack_release(self):
         """Get openstack release.

=== modified file 'charmhelpers/contrib/openstack/amulet/utils.py'
--- charmhelpers/contrib/openstack/amulet/utils.py	2015-09-28 18:47:55 +0000
+++ charmhelpers/contrib/openstack/amulet/utils.py	2015-10-21 03:27:04 +0000
@@ -18,6 +18,7 @@
 import json
 import logging
 import os
+import re
 import six
 import time
 import urllib
@@ -604,7 +605,22 @@
                            '{}'.format(sample_type, samples))
             return None
 
-# rabbitmq/amqp specific helpers:
+    # rabbitmq/amqp specific helpers:
+
+    def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
+        """Wait for rmq units extended status to show cluster readiness,
+        after an optional initial sleep period.  Initial sleep is likely
+        necessary to be effective following a config change, as status
+        message may not instantly update to non-ready."""
+
+        if init_sleep:
+            time.sleep(init_sleep)
+
+        message = re.compile('^Unit is ready and clustered$')
+        deployment._auto_wait_for_status(message=message,
+                                         timeout=timeout,
+                                         include_only=['rabbitmq-server'])
+
     def add_rmq_test_user(self, sentry_units,
                           username="testuser1", password="changeme"):
         """Add a test user via the first rmq juju unit, check connection as
@@ -805,7 +821,10 @@
         if port:
             config['ssl_port'] = port
 
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0
@@ -832,7 +851,10 @@
 
         # Disable RMQ SSL
         config = {'ssl': 'off'}
-        deployment.configure('rabbitmq-server', config)
+        deployment.d.configure('rabbitmq-server', config)
+
+        # Wait for unit status
+        self.rmq_wait_for_cluster(deployment)
 
         # Confirm
         tries = 0