Merge lp:~corey.bryant/charms/trusty/keystone/sync-charm-helpers2 into lp:~openstack-charmers-archive/charms/trusty/keystone/next

[Corey Bryant]

The amulet tests are broken without this.

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-07-25 08:13:49 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-07-30 15:28:46 +0000
@@ -62,6 +62,15 @@
     return peers
 
 
+def peer_ips(peer_relation='cluster', addr_key='private-address'):
+    '''Return a dict of peers and their private-address'''
+    peers = {}
+    for r_id in relation_ids(peer_relation):
+        for unit in relation_list(r_id):
+            peers[unit] = relation_get(addr_key, rid=r_id, unit=unit)
+    return peers
+
+
 def oldest_peer(peers):
     local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1])
     for peer in peers:

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-07-25 08:13:49 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-07-30 15:28:46 +0000
@@ -4,8 +4,11 @@
 
 
 class OpenStackAmuletDeployment(AmuletDeployment):
-    """This class inherits from AmuletDeployment and has additional support
-       that is specifically for use by OpenStack charms."""
+    """OpenStack amulet deployment.
+
+       This class inherits from AmuletDeployment and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, series=None, openstack=None, source=None):
         """Initialize the deployment environment."""
@@ -40,11 +43,14 @@
             self.d.configure(service, config)
 
     def _get_openstack_release(self):
-        """Return an integer representing the enum value of the openstack
-           release."""
-        self.precise_essex, self.precise_folsom, self.precise_grizzly, \
-            self.precise_havana, self.precise_icehouse, \
-            self.trusty_icehouse = range(6)
+        """Get openstack release.
+
+           Return an integer representing the enum value of the openstack
+           release.
+           """
+        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+         self.precise_havana, self.precise_icehouse,
+         self.trusty_icehouse) = range(6)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2014-07-25 08:13:49 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2014-07-30 15:28:46 +0000
@@ -16,8 +16,11 @@
 
 
 class OpenStackAmuletUtils(AmuletUtils):
-    """This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms."""
+    """OpenStack amulet utilities.
+
+       This class inherits from AmuletUtils and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, log_level=ERROR):
         """Initialize the deployment environment."""
@@ -25,13 +28,17 @@
 
     def validate_endpoint_data(self, endpoints, admin_port, internal_port,
                                public_port, expected):
-        """Validate actual endpoint data vs expected endpoint data. The ports
-           are used to find the matching endpoint."""
+        """Validate endpoint data.
+
+           Validate actual endpoint data vs expected endpoint data. The ports
+           are used to find the matching endpoint.
+           """
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
-            if admin_port in ep.adminurl and internal_port in ep.internalurl \
-               and public_port in ep.publicurl:
+            if (admin_port in ep.adminurl and
+                    internal_port in ep.internalurl and
+                    public_port in ep.publicurl):
                 found = True
                 actual = {'id': ep.id,
                           'region': ep.region,
@@ -47,8 +54,11 @@
             return 'endpoint not found'
 
     def validate_svc_catalog_endpoint_data(self, expected, actual):
-        """Validate a list of actual service catalog endpoints vs a list of
-           expected service catalog endpoints."""
+        """Validate service catalog endpoint data.
+
+           Validate a list of actual service catalog endpoints vs a list of
+           expected service catalog endpoints.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in expected.iteritems():
             if k in actual:
@@ -60,8 +70,11 @@
         return ret
 
     def validate_tenant_data(self, expected, actual):
-        """Validate a list of actual tenant data vs list of expected tenant
-           data."""
+        """Validate tenant data.
+
+           Validate a list of actual tenant data vs list of expected tenant
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -78,8 +91,11 @@
         return ret
 
     def validate_role_data(self, expected, actual):
-        """Validate a list of actual role data vs a list of expected role
-           data."""
+        """Validate role data.
+
+           Validate a list of actual role data vs a list of expected role
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -95,8 +111,11 @@
         return ret
 
     def validate_user_data(self, expected, actual):
-        """Validate a list of actual user data vs a list of expected user
-           data."""
+        """Validate user data.
+
+           Validate a list of actual user data vs a list of expected user
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,21 +133,24 @@
         return ret
 
     def validate_flavor_data(self, expected, actual):
-        """Validate a list of actual flavors vs a list of expected flavors."""
+        """Validate flavor data.
+
+           Validate a list of actual flavors vs a list of expected flavors.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
-        """Return True if tenant exists"""
+        """Return True if tenant exists."""
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
-        service_ip = \
-            keystone_sentry.relation('shared-db',
-                                     'mysql:shared-db')['private-address']
+        unit = keystone_sentry
+        service_ip = unit.relation('shared-db',
+                                   'mysql:shared-db')['private-address']
         ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
         return keystone_client.Client(username=user, password=password,
                                       tenant_name=tenant, auth_url=ep)
@@ -177,12 +199,40 @@
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
+        count = 1
+        status = image.status
+        while status != 'active' and count < 10:
+            time.sleep(3)
+            image = glance.images.get(image.id)
+            status = image.status
+            self.log.debug('image status: {}'.format(status))
+            count += 1
+
+        if status != 'active':
+            self.log.error('image creation timed out')
+            return None
+
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+        num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
+        count = 1
+        num_after = len(list(glance.images.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(glance.images.list()))
+            self.log.debug('number of images: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('image deletion timed out')
+            return False
+
+        return True
+
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
         image = nova.images.find(name=image_name)
@@ -199,11 +249,27 @@
             self.log.debug('instance status: {}'.format(status))
             count += 1
 
-        if status == 'BUILD':
+        if status != 'ACTIVE':
+            self.log.error('instance creation timed out')
             return None
 
         return instance
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+        num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
+
+        count = 1
+        num_after = len(list(nova.servers.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(nova.servers.list()))
+            self.log.debug('number of instances: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('instance deletion timed out')
+            return False
+
+        return True

=== modified file 'tests/charmhelpers/contrib/amulet/deployment.py'
--- tests/charmhelpers/contrib/amulet/deployment.py	2014-07-25 08:13:49 +0000
+++ tests/charmhelpers/contrib/amulet/deployment.py	2014-07-30 15:28:46 +0000
@@ -1,9 +1,14 @@
 import amulet
 
+import os
+
 
 class AmuletDeployment(object):
-    """This class provides generic Amulet deployment and test runner
-       methods."""
+    """Amulet deployment.
+
+       This class provides generic Amulet deployment and test runner
+       methods.
+       """
 
     def __init__(self, series=None):
         """Initialize the deployment environment."""
@@ -16,11 +21,19 @@
             self.d = amulet.Deployment()
 
     def _add_services(self, this_service, other_services):
-        """Add services to the deployment where this_service is the local charm
+        """Add services.
+
+           Add services to the deployment where this_service is the local charm
            that we're focused on testing and other_services are the other
-           charms that come from the charm store."""
+           charms that come from the charm store.
+           """
         name, units = range(2)
-        self.this_service = this_service[name]
+
+        if this_service[name] != os.path.basename(os.getcwd()):
+            s = this_service[name]
+            msg = "The charm's root directory name needs to be {}".format(s)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
         self.d.add(this_service[name], units=this_service[units])
 
         for svc in other_services:
@@ -45,10 +58,10 @@
         """Deploy environment and wait for all hooks to finish executing."""
         try:
             self.d.setup()
-            self.d.sentry.wait()
+            self.d.sentry.wait(timeout=900)
         except amulet.helpers.TimeoutError:
             amulet.raise_status(amulet.FAIL, msg="Deployment timed out")
-        except:
+        except Exception:
             raise
 
     def run_tests(self):

=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
--- tests/charmhelpers/contrib/amulet/utils.py	2014-07-25 08:13:49 +0000
+++ tests/charmhelpers/contrib/amulet/utils.py	2014-07-30 15:28:46 +0000
@@ -3,12 +3,15 @@
 import logging
 import re
 import sys
-from time import sleep
+import time
 
 
 class AmuletUtils(object):
-    """This class provides common utility functions that are used by Amulet
-       tests."""
+    """Amulet utilities.
+
+       This class provides common utility functions that are used by Amulet
+       tests.
+       """
 
     def __init__(self, log_level=logging.ERROR):
         self.log = self.get_logger(level=log_level)
@@ -17,8 +20,8 @@
         """Get a logger object that will log to stdout."""
         log = logging
         logger = log.getLogger(name)
-        fmt = \
-            log.Formatter("%(asctime)s %(funcName)s %(levelname)s: %(message)s")
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
 
         handler = log.StreamHandler(stream=sys.stdout)
         handler.setLevel(level)
@@ -38,7 +41,7 @@
     def valid_url(self, url):
         p = re.compile(
             r'^(?:http|ftp)s?://'
-            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # flake8: noqa
+            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # noqa
             r'localhost|'
             r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
             r'(?::\d+)?'
@@ -50,8 +53,11 @@
             return False
 
     def validate_services(self, commands):
-        """Verify the specified services are running on the corresponding
-           service units."""
+        """Validate services.
+
+           Verify the specified services are running on the corresponding
+           service units.
+           """
         for k, v in commands.iteritems():
             for cmd in v:
                 output, code = k.run(cmd)
@@ -66,9 +72,13 @@
         config.readfp(io.StringIO(file_contents))
         return config
 
-    def validate_config_data(self, sentry_unit, config_file, section, expected):
-        """Verify that the specified section of the config file contains
-           the expected option key:value pairs."""
+    def validate_config_data(self, sentry_unit, config_file, section,
+                             expected):
+        """Validate config file data.
+
+           Verify that the specified section of the config file contains
+           the expected option key:value pairs.
+           """
         config = self._get_config(sentry_unit, config_file)
 
         if section != 'DEFAULT' and not config.has_section(section):
@@ -78,20 +88,23 @@
             if not config.has_option(section, k):
                 return "section [{}] is missing option {}".format(section, k)
             if config.get(section, k) != expected[k]:
-                return "section [{}] {}:{} != expected {}:{}".format(section,
-                       k, config.get(section, k), k, expected[k])
+                return "section [{}] {}:{} != expected {}:{}".format(
+                       section, k, config.get(section, k), k, expected[k])
         return None
 
     def _validate_dict_data(self, expected, actual):
-        """Compare expected dictionary data vs actual dictionary data.
+        """Validate dictionary data.
+
+           Compare expected dictionary data vs actual dictionary data.
            The values in the 'expected' dictionary can be strings, bools, ints,
            longs, or can be a function that evaluate a variable and returns a
-           bool."""
+           bool.
+           """
         for k, v in expected.iteritems():
             if k in actual:
-                if isinstance(v, basestring) or \
-                   isinstance(v, bool) or \
-                   isinstance(v, (int, long)):
+                if (isinstance(v, basestring) or
+                        isinstance(v, bool) or
+                        isinstance(v, (int, long))):
                     if v != actual[k]:
                         return "{}:{}".format(k, actual[k])
                 elif not v(actual[k]):
@@ -114,7 +127,7 @@
         return None
 
     def not_null(self, string):
-        if string != None:
+        if string is not None:
             return True
         else:
             return False
@@ -128,9 +141,12 @@
         return sentry_unit.directory_stat(directory)['mtime']
 
     def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
-        """Determine start time of the process based on the last modification
+        """Get process' start time.
+
+           Determine start time of the process based on the last modification
            time of the /proc/pid directory. If pgrep_full is True, the process
-           name is matched against the full command line."""
+           name is matched against the full command line.
+           """
         if pgrep_full:
             cmd = 'pgrep -o -f {}'.format(service)
         else:
@@ -139,13 +155,16 @@
         return self._get_dir_mtime(sentry_unit, proc_dir)
 
     def service_restarted(self, sentry_unit, service, filename,
-                          pgrep_full=False):
-        """Compare a service's start time vs a file's last modification time
+                          pgrep_full=False, sleep_time=20):
+        """Check if service was restarted.
+
+           Compare a service's start time vs a file's last modification time
            (such as a config file for that service) to determine if the service
-           has been restarted."""
-        sleep(10)
-        if self._get_proc_start_time(sentry_unit, service, pgrep_full) >= \
-           self._get_file_mtime(sentry_unit, filename):
+           has been restarted.
+           """
+        time.sleep(sleep_time)
+        if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
+                self._get_file_mtime(sentry_unit, filename)):
             return True
         else:
             return False

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-07-25 08:13:49 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-07-30 15:28:46 +0000
@@ -4,8 +4,11 @@
 
 
 class OpenStackAmuletDeployment(AmuletDeployment):
-    """This class inherits from AmuletDeployment and has additional support
-       that is specifically for use by OpenStack charms."""
+    """OpenStack amulet deployment.
+
+       This class inherits from AmuletDeployment and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, series=None, openstack=None, source=None):
         """Initialize the deployment environment."""
@@ -40,11 +43,14 @@
             self.d.configure(service, config)
 
     def _get_openstack_release(self):
-        """Return an integer representing the enum value of the openstack
-           release."""
-        self.precise_essex, self.precise_folsom, self.precise_grizzly, \
-            self.precise_havana, self.precise_icehouse, \
-            self.trusty_icehouse = range(6)
+        """Get openstack release.
+
+           Return an integer representing the enum value of the openstack
+           release.
+           """
+        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+         self.precise_havana, self.precise_icehouse,
+         self.trusty_icehouse) = range(6)
         releases = {
             ('precise', None): self.precise_essex,
             ('precise', 'cloud:precise-folsom'): self.precise_folsom,

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-07-25 08:13:49 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-07-30 15:28:46 +0000
@@ -16,8 +16,11 @@
 
 
 class OpenStackAmuletUtils(AmuletUtils):
-    """This class inherits from AmuletUtils and has additional support
-       that is specifically for use by OpenStack charms."""
+    """OpenStack amulet utilities.
+
+       This class inherits from AmuletUtils and has additional support
+       that is specifically for use by OpenStack charms.
+       """
 
     def __init__(self, log_level=ERROR):
         """Initialize the deployment environment."""
@@ -25,13 +28,17 @@
 
     def validate_endpoint_data(self, endpoints, admin_port, internal_port,
                                public_port, expected):
-        """Validate actual endpoint data vs expected endpoint data. The ports
-           are used to find the matching endpoint."""
+        """Validate endpoint data.
+
+           Validate actual endpoint data vs expected endpoint data. The ports
+           are used to find the matching endpoint.
+           """
         found = False
         for ep in endpoints:
             self.log.debug('endpoint: {}'.format(repr(ep)))
-            if admin_port in ep.adminurl and internal_port in ep.internalurl \
-               and public_port in ep.publicurl:
+            if (admin_port in ep.adminurl and
+                    internal_port in ep.internalurl and
+                    public_port in ep.publicurl):
                 found = True
                 actual = {'id': ep.id,
                           'region': ep.region,
@@ -47,8 +54,11 @@
             return 'endpoint not found'
 
     def validate_svc_catalog_endpoint_data(self, expected, actual):
-        """Validate a list of actual service catalog endpoints vs a list of
-           expected service catalog endpoints."""
+        """Validate service catalog endpoint data.
+
+           Validate a list of actual service catalog endpoints vs a list of
+           expected service catalog endpoints.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for k, v in expected.iteritems():
             if k in actual:
@@ -60,8 +70,11 @@
         return ret
 
     def validate_tenant_data(self, expected, actual):
-        """Validate a list of actual tenant data vs list of expected tenant
-           data."""
+        """Validate tenant data.
+
+           Validate a list of actual tenant data vs list of expected tenant
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -78,8 +91,11 @@
         return ret
 
     def validate_role_data(self, expected, actual):
-        """Validate a list of actual role data vs a list of expected role
-           data."""
+        """Validate role data.
+
+           Validate a list of actual role data vs a list of expected role
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -95,8 +111,11 @@
         return ret
 
     def validate_user_data(self, expected, actual):
-        """Validate a list of actual user data vs a list of expected user
-           data."""
+        """Validate user data.
+
+           Validate a list of actual user data vs a list of expected user
+           data.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         for e in expected:
             found = False
@@ -114,21 +133,24 @@
         return ret
 
     def validate_flavor_data(self, expected, actual):
-        """Validate a list of actual flavors vs a list of expected flavors."""
+        """Validate flavor data.
+
+           Validate a list of actual flavors vs a list of expected flavors.
+           """
         self.log.debug('actual: {}'.format(repr(actual)))
         act = [a.name for a in actual]
         return self._validate_list_data(expected, act)
 
     def tenant_exists(self, keystone, tenant):
-        """Return True if tenant exists"""
+        """Return True if tenant exists."""
         return tenant in [t.name for t in keystone.tenants.list()]
 
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
                                     tenant):
         """Authenticates admin user with the keystone admin endpoint."""
-        service_ip = \
-            keystone_sentry.relation('shared-db',
-                                     'mysql:shared-db')['private-address']
+        unit = keystone_sentry
+        service_ip = unit.relation('shared-db',
+                                   'mysql:shared-db')['private-address']
         ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
         return keystone_client.Client(username=user, password=password,
                                       tenant_name=tenant, auth_url=ep)
@@ -177,12 +199,40 @@
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
+        count = 1
+        status = image.status
+        while status != 'active' and count < 10:
+            time.sleep(3)
+            image = glance.images.get(image.id)
+            status = image.status
+            self.log.debug('image status: {}'.format(status))
+            count += 1
+
+        if status != 'active':
+            self.log.error('image creation timed out')
+            return None
+
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
+        num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
+        count = 1
+        num_after = len(list(glance.images.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(glance.images.list()))
+            self.log.debug('number of images: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('image deletion timed out')
+            return False
+
+        return True
+
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
         image = nova.images.find(name=image_name)
@@ -199,11 +249,27 @@
             self.log.debug('instance status: {}'.format(status))
             count += 1
 
-        if status == 'BUILD':
+        if status != 'ACTIVE':
+            self.log.error('instance creation timed out')
             return None
 
         return instance
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
+        num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
+
+        count = 1
+        num_after = len(list(nova.servers.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(nova.servers.list()))
+            self.log.debug('number of instances: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('instance deletion timed out')
+            return False
+
+        return True