Merge lp:~corey.bryant/charms/trusty/keystone/contrib.python.packages into lp:~openstack-charmers-archive/charms/trusty/keystone/next

[Corey Bryant]

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #163 keystone-next for corey.bryant mp244323
    LINT OK: passed

Build: http://10.230.18.80:8080/job/charm_lint_check/163/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #126 keystone-next for corey.bryant mp244323
    UNIT FAIL: unit-test failed

UNIT Results (max last 2 lines):
  FAILED (errors=3)
  make: *** [unit_test] Error 1

Full unit test output: pastebin not avail., cmd error
Build: http://10.230.18.80:8080/job/charm_unit_test/126/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #76 keystone-next for corey.bryant mp244323
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
  ERROR subprocess encountered error code 2
  make: *** [test] Error 2

Full amulet test output: pastebin not avail., cmd error
Build: http://10.230.18.80:8080/job/charm_amulet_test/76/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #185 keystone-next for corey.bryant mp244323
    LINT OK: passed

Build: http://10.230.18.80:8080/job/charm_lint_check/185/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #148 keystone-next for corey.bryant mp244323
    UNIT OK: passed

Build: http://10.230.18.80:8080/job/charm_unit_test/148/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #103 keystone-next for corey.bryant mp244323
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
  ERROR subprocess encountered error code 1
  make: *** [test] Error 1

Full amulet test output: pastebin not avail., cmd error
Build: http://10.230.18.80:8080/job/charm_amulet_test/103/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #209 keystone-next for corey.bryant mp244323
    LINT OK: passed

Build: http://10.230.18.80:8080/job/charm_lint_check/209/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #172 keystone-next for corey.bryant mp244323
    UNIT OK: passed

Build: http://10.230.18.80:8080/job/charm_unit_test/172/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #127 keystone-next for corey.bryant mp244323
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
  ERROR subprocess encountered error code 1
  make: *** [test] Error 1

Full amulet test output: pastebin not avail., cmd error
Build: http://10.230.18.80:8080/job/charm_amulet_test/127/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #227 keystone-next for corey.bryant mp244323
    LINT OK: passed

Build: http://10.230.18.80:8080/job/charm_lint_check/227/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #190 keystone-next for corey.bryant mp244323
    UNIT OK: passed

Build: http://10.230.18.80:8080/job/charm_unit_test/190/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #144 keystone-next for corey.bryant mp244323
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
  ERROR subprocess encountered error code 1
  make: *** [test] Error 1

Full amulet test output: pastebin not avail., cmd error
Build: http://10.230.18.80:8080/job/charm_amulet_test/144/

=== modified file 'charm-helpers-hooks.yaml'
--- charm-helpers-hooks.yaml	2014-10-27 02:20:30 +0000
+++ charm-helpers-hooks.yaml	2014-12-11 17:56:34 +0000
@@ -12,3 +12,4 @@
     - payload.execd
     - contrib.peerstorage
     - contrib.network.ip
+    - contrib.python.packages
 
=== added file 'hooks/charmhelpers/__init__.py'
--- hooks/charmhelpers/__init__.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/__init__.py	2014-12-11 17:56:34 +0000
@@ -0,0 +1,22 @@
+# Bootstrap charm-helpers, installing its dependencies if necessary using
+# only standard libraries.
+import subprocess
+import sys
+
+try:
+    import six  # flake8: noqa
+except ImportError:
+    if sys.version_info.major == 2:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
+    else:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
+    import six  # flake8: noqa
+
+try:
+    import yaml  # flake8: noqa
+except ImportError:
+    if sys.version_info.major == 2:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
+    else:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
+    import yaml  # flake8: noqa
 
=== removed file 'hooks/charmhelpers/__init__.py'
=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-10-06 20:51:06 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-12-11 17:56:34 +0000
@@ -13,9 +13,10 @@
 
 import subprocess
 import os
-
 from socket import gethostname as get_unit_hostname
 
+import six
+
 from charmhelpers.core.hookenv import (
     log,
     relation_ids,
@@ -77,7 +78,7 @@
         "show", resource
     ]
     try:
-        status = subprocess.check_output(cmd)
+        status = subprocess.check_output(cmd).decode('UTF-8')
     except subprocess.CalledProcessError:
         return False
     else:
@@ -150,34 +151,42 @@
     return False
 
 
-def determine_api_port(public_port):
+def determine_api_port(public_port, singlenode_mode=False):
     '''
     Determine correct API server listening port based on
     existence of HTTPS reverse proxy and/or haproxy.
 
     public_port: int: standard public port for given service
 
+    singlenode_mode: boolean: Shuffle ports when only a single unit is present
+
     returns: int: the correct listening port for the API service
     '''
     i = 0
-    if len(peer_units()) > 0 or is_clustered():
+    if singlenode_mode:
+        i += 1
+    elif len(peer_units()) > 0 or is_clustered():
         i += 1
     if https():
         i += 1
     return public_port - (i * 10)
 
 
-def determine_apache_port(public_port):
+def determine_apache_port(public_port, singlenode_mode=False):
     '''
     Description: Determine correct apache listening port based on public IP +
     state of the cluster.
 
     public_port: int: standard public port for given service
 
+    singlenode_mode: boolean: Shuffle ports when only a single unit is present
+
     returns: int: the correct listening port for the HAProxy service
     '''
     i = 0
-    if len(peer_units()) > 0 or is_clustered():
+    if singlenode_mode:
+        i += 1
+    elif len(peer_units()) > 0 or is_clustered():
         i += 1
     return public_port - (i * 10)
 
@@ -197,7 +206,7 @@
     for setting in settings:
         conf[setting] = config_get(setting)
     missing = []
-    [missing.append(s) for s, v in conf.iteritems() if v is None]
+    [missing.append(s) for s, v in six.iteritems(conf) if v is None]
     if missing:
         log('Insufficient config data to configure hacluster.', level=ERROR)
         raise HAIncompleteConfig
 
=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	2014-10-27 02:14:51 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2014-12-11 17:56:34 +0000
@@ -1,14 +1,12 @@
 import glob
 import re
 import subprocess
-import sys
 
 from functools import partial
 
 from charmhelpers.core.hookenv import unit_get
 from charmhelpers.fetch import apt_install
 from charmhelpers.core.hookenv import (
-    ERROR,
     log
 )
 
@@ -33,31 +31,28 @@
                          network)
 
 
+def no_ip_found_error_out(network):
+    errmsg = ("No IP address found in network: %s" % network)
+    raise ValueError(errmsg)
+
+
 def get_address_in_network(network, fallback=None, fatal=False):
-    """
-    Get an IPv4 or IPv6 address within the network from the host.
+    """Get an IPv4 or IPv6 address within the network from the host.
 
     :param network (str): CIDR presentation format. For example,
         '192.168.1.0/24'.
     :param fallback (str): If no address is found, return fallback.
     :param fatal (boolean): If no address is found, fallback is not
         set and fatal is True then exit(1).
-
     """
-
-    def not_found_error_out():
-        log("No IP address found in network: %s" % network,
-            level=ERROR)
-        sys.exit(1)
-
     if network is None:
         if fallback is not None:
             return fallback
+
+        if fatal:
+            no_ip_found_error_out(network)
         else:
-            if fatal:
-                not_found_error_out()
-            else:
-                return None
+            return None
 
     _validate_cidr(network)
     network = netaddr.IPNetwork(network)
@@ -69,6 +64,7 @@
             cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
             if cidr in network:
                 return str(cidr.ip)
+
         if network.version == 6 and netifaces.AF_INET6 in addresses:
             for addr in addresses[netifaces.AF_INET6]:
                 if not addr['addr'].startswith('fe80'):
@@ -81,20 +77,20 @@
         return fallback
 
     if fatal:
-        not_found_error_out()
+        no_ip_found_error_out(network)
 
     return None
 
 
 def is_ipv6(address):
-    '''Determine whether provided address is IPv6 or not'''
+    """Determine whether provided address is IPv6 or not."""
     try:
         address = netaddr.IPAddress(address)
     except netaddr.AddrFormatError:
         # probably a hostname - so not an address at all!
         return False
-    else:
-        return address.version == 6
+
+    return address.version == 6
 
 
 def is_address_in_network(network, address):
@@ -112,11 +108,13 @@
     except (netaddr.core.AddrFormatError, ValueError):
         raise ValueError("Network (%s) is not in CIDR presentation format" %
                          network)
+
     try:
         address = netaddr.IPAddress(address)
     except (netaddr.core.AddrFormatError, ValueError):
         raise ValueError("Address (%s) is not in correct presentation format" %
                          address)
+
     if address in network:
         return True
     else:
@@ -146,6 +144,7 @@
                     return iface
                 else:
                     return addresses[netifaces.AF_INET][0][key]
+
         if address.version == 6 and netifaces.AF_INET6 in addresses:
             for addr in addresses[netifaces.AF_INET6]:
                 if not addr['addr'].startswith('fe80'):
@@ -159,40 +158,42 @@
                             return str(cidr).split('/')[1]
                         else:
                             return addr[key]
+
     return None
 
 
 get_iface_for_address = partial(_get_for_address, key='iface')
 
+
 get_netmask_for_address = partial(_get_for_address, key='netmask')
 
 
 def format_ipv6_addr(address):
-    """
-    IPv6 needs to be wrapped with [] in url link to parse correctly.
+    """If address is IPv6, wrap it in '[]' otherwise return None.
+
+    This is required by most configuration files when specifying IPv6
+    addresses.
     """
     if is_ipv6(address):
-        address = "[%s]" % address
-    else:
-        address = None
+        return "[%s]" % address
 
-    return address
+    return None
 
 
 def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
                    fatal=True, exc_list=None):
-    """
-    Return the assigned IP address for a given interface, if any, or [].
-    """
+    """Return the assigned IP address for a given interface, if any."""
     # Extract nic if passed /dev/ethX
     if '/' in iface:
         iface = iface.split('/')[-1]
+
     if not exc_list:
         exc_list = []
+
     try:
         inet_num = getattr(netifaces, inet_type)
     except AttributeError:
-        raise Exception('Unknown inet type ' + str(inet_type))
+        raise Exception("Unknown inet type '%s'" % str(inet_type))
 
     interfaces = netifaces.interfaces()
     if inc_aliases:
@@ -200,15 +201,18 @@
         for _iface in interfaces:
             if iface == _iface or _iface.split(':')[0] == iface:
                 ifaces.append(_iface)
+
         if fatal and not ifaces:
             raise Exception("Invalid interface '%s'" % iface)
+
         ifaces.sort()
     else:
         if iface not in interfaces:
             if fatal:
-                raise Exception("%s not found " % (iface))
+                raise Exception("Interface '%s' not found " % (iface))
             else:
                 return []
+
         else:
             ifaces = [iface]
 
@@ -219,10 +223,13 @@
             for entry in net_info[inet_num]:
                 if 'addr' in entry and entry['addr'] not in exc_list:
                     addresses.append(entry['addr'])
+
     if fatal and not addresses:
         raise Exception("Interface '%s' doesn't have any %s addresses." %
                         (iface, inet_type))
-    return addresses
+
+    return sorted(addresses)
+
 
 get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
 
@@ -239,6 +246,7 @@
                 raw = re.match(ll_key, _addr)
                 if raw:
                     _addr = raw.group(1)
+
                 if _addr == addr:
                     log("Address '%s' is configured on iface '%s'" %
                         (addr, iface))
@@ -249,8 +257,9 @@
 
 
 def sniff_iface(f):
-    """If no iface provided, inject net iface inferred from unit private
-    address.
+    """Ensure decorated function is called with a value for iface.
+
+    If no iface provided, inject net iface inferred from unit private address.
     """
     def iface_sniffer(*args, **kwargs):
         if not kwargs.get('iface', None):
@@ -293,7 +302,7 @@
         if global_addrs:
             # Make sure any found global addresses are not temporary
             cmd = ['ip', 'addr', 'show', iface]
-            out = subprocess.check_output(cmd)
+            out = subprocess.check_output(cmd).decode('UTF-8')
             if dynamic_only:
                 key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
             else:
@@ -315,33 +324,28 @@
                 return addrs
 
     if fatal:
-        raise Exception("Interface '%s' doesn't have a scope global "
+        raise Exception("Interface '%s' does not have a scope global "
                         "non-temporary ipv6 address." % iface)
 
     return []
 
 
 def get_bridges(vnic_dir='/sys/devices/virtual/net'):
-    """
-    Return a list of bridges on the system or []
-    """
-    b_rgex = vnic_dir + '/*/bridge'
-    return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)]
+    """Return a list of bridges on the system."""
+    b_regex = "%s/*/bridge" % vnic_dir
+    return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_regex)]
 
 
 def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'):
-    """
-    Return a list of nics comprising a given bridge on the system or []
-    """
-    brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge)
-    return [x.split('/')[-1] for x in glob.glob(brif_rgex)]
+    """Return a list of nics comprising a given bridge on the system."""
+    brif_regex = "%s/%s/brif/*" % (vnic_dir, bridge)
+    return [x.split('/')[-1] for x in glob.glob(brif_regex)]
 
 
 def is_bridge_member(nic):
-    """
-    Check if a given nic is a member of a bridge
-    """
+    """Check if a given nic is a member of a bridge."""
     for bridge in get_bridges():
         if nic in get_bridge_nics(bridge):
             return True
+
     return False
 
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-10-06 20:51:06 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-12-11 17:56:34 +0000
@@ -1,3 +1,4 @@
+import six
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -69,7 +70,7 @@
 
     def _configure_services(self, configs):
         """Configure all of the services."""
-        for service, config in configs.iteritems():
+        for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
     def _get_openstack_release(self):
 
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2014-10-06 20:51:06 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2014-12-11 17:56:34 +0000
@@ -7,6 +7,8 @@
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
 
+import six
+
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -60,7 +62,7 @@
            expected service catalog endpoints.
            """
         self.log.debug('actual: {}'.format(repr(actual)))
-        for k, v in expected.iteritems():
+        for k, v in six.iteritems(expected):
             if k in actual:
                 ret = self._validate_dict_data(expected[k][0], actual[k][0])
                 if ret:
 
=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-11-14 02:16:38 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-12-11 17:56:34 +0000
@@ -1,18 +1,15 @@
 import json
 import os
 import time
-
 from base64 import b64decode
+from subprocess import check_call
 
-from subprocess import (
-    check_call
-)
+import six
 
 from charmhelpers.fetch import (
     apt_install,
     filter_installed_packages,
 )
-
 from charmhelpers.core.hookenv import (
     config,
     is_relation_made,
@@ -24,44 +21,40 @@
     relation_set,
     unit_get,
     unit_private_ip,
+    DEBUG,
+    INFO,
+    WARNING,
     ERROR,
-    DEBUG
 )
-
 from charmhelpers.core.host import (
     mkdir,
-    write_file
+    write_file,
 )
-
 from charmhelpers.contrib.hahelpers.cluster import (
     determine_apache_port,
     determine_api_port,
     https,
-    is_clustered
+    is_clustered,
 )
-
 from charmhelpers.contrib.hahelpers.apache import (
     get_cert,
     get_ca_cert,
     install_ca_cert,
 )
-
 from charmhelpers.contrib.openstack.neutron import (
     neutron_plugin_attribute,
 )
-
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
     get_ipv6_addr,
     get_netmask_for_address,
     format_ipv6_addr,
-    is_address_in_network
+    is_address_in_network,
 )
+from charmhelpers.contrib.openstack.utils import get_host_ip
 
-from charmhelpers.contrib.openstack.utils import (
-    get_host_ip,
-)
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
+ADDRESS_TYPES = ['admin', 'internal', 'public']
 
 
 class OSContextError(Exception):
@@ -69,7 +62,7 @@
 
 
 def ensure_packages(packages):
-    '''Install but do not upgrade required plugin packages'''
+    """Install but do not upgrade required plugin packages."""
     required = filter_installed_packages(packages)
     if required:
         apt_install(required, fatal=True)
@@ -77,20 +70,27 @@
 
 def context_complete(ctxt):
     _missing = []
-    for k, v in ctxt.iteritems():
+    for k, v in six.iteritems(ctxt):
         if v is None or v == '':
             _missing.append(k)
+
     if _missing:
-        log('Missing required data: %s' % ' '.join(_missing), level='INFO')
+        log('Missing required data: %s' % ' '.join(_missing), level=INFO)
         return False
+
     return True
 
 
 def config_flags_parser(config_flags):
+    """Parses config flags string into dict.
+
+    The provided config_flags string may be a list of comma-separated values
+    which themselves may be comma-separated list of values.
+    """
     if config_flags.find('==') >= 0:
-        log("config_flags is not in expected format (key=value)",
-            level=ERROR)
+        log("config_flags is not in expected format (key=value)", level=ERROR)
         raise OSContextError
+
     # strip the following from each value.
     post_strippers = ' ,'
     # we strip any leading/trailing '=' or ' ' from the string then
@@ -98,7 +98,7 @@
     split = config_flags.strip(' =').split('=')
     limit = len(split)
     flags = {}
-    for i in xrange(0, limit - 1):
+    for i in range(0, limit - 1):
         current = split[i]
         next = split[i + 1]
         vindex = next.rfind(',')
@@ -113,17 +113,18 @@
             # if this not the first entry, expect an embedded key.
             index = current.rfind(',')
             if index < 0:
-                log("invalid config value(s) at index %s" % (i),
-                    level=ERROR)
+                log("Invalid config value(s) at index %s" % (i), level=ERROR)
                 raise OSContextError
             key = current[index + 1:]
 
         # Add to collection.
         flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
+
     return flags
 
 
 class OSContextGenerator(object):
+    """Base class for all context generators."""
     interfaces = []
 
     def __call__(self):
@@ -135,11 +136,11 @@
 
     def __init__(self,
                  database=None, user=None, relation_prefix=None, ssl_dir=None):
-        '''
-        Allows inspecting relation for settings prefixed with relation_prefix.
-        This is useful for parsing access for multiple databases returned via
-        the shared-db interface (eg, nova_password, quantum_password)
-        '''
+        """Allows inspecting relation for settings prefixed with
+        relation_prefix. This is useful for parsing access for multiple
+        databases returned via the shared-db interface (eg, nova_password,
+        quantum_password)
+        """
         self.relation_prefix = relation_prefix
         self.database = database
         self.user = user
@@ -149,9 +150,8 @@
         self.database = self.database or config('database')
         self.user = self.user or config('database-user')
         if None in [self.database, self.user]:
-            log('Could not generate shared_db context. '
-                'Missing required charm config options. '
-                '(database name and user)')
+            log("Could not generate shared_db context. Missing required charm "
+                "config options. (database name and user)", level=ERROR)
             raise OSContextError
 
         ctxt = {}
@@ -204,23 +204,24 @@
     def __call__(self):
         self.database = self.database or config('database')
         if self.database is None:
-            log('Could not generate postgresql_db context. '
-                'Missing required charm config options. '
-                '(database name)')
+            log('Could not generate postgresql_db context. Missing required '
+                'charm config options. (database name)', level=ERROR)
             raise OSContextError
+
         ctxt = {}
-
         for rid in relation_ids(self.interfaces[0]):
             for unit in related_units(rid):
-                ctxt = {
-                    'database_host': relation_get('host', rid=rid, unit=unit),
-                    'database': self.database,
-                    'database_user': relation_get('user', rid=rid, unit=unit),
-                    'database_password': relation_get('password', rid=rid, unit=unit),
-                    'database_type': 'postgresql',
-                }
+                rel_host = relation_get('host', rid=rid, unit=unit)
+                rel_user = relation_get('user', rid=rid, unit=unit)
+                rel_passwd = relation_get('password', rid=rid, unit=unit)
+                ctxt = {'database_host': rel_host,
+                        'database': self.database,
+                        'database_user': rel_user,
+                        'database_password': rel_passwd,
+                        'database_type': 'postgresql'}
                 if context_complete(ctxt):
                     return ctxt
+
         return {}
 
 
@@ -229,23 +230,29 @@
         ca_path = os.path.join(ssl_dir, 'db-client.ca')
         with open(ca_path, 'w') as fh:
             fh.write(b64decode(rdata['ssl_ca']))
+
         ctxt['database_ssl_ca'] = ca_path
     elif 'ssl_ca' in rdata:
-        log("Charm not setup for ssl support but ssl ca found")
+        log("Charm not setup for ssl support but ssl ca found", level=INFO)
         return ctxt
+
     if 'ssl_cert' in rdata:
         cert_path = os.path.join(
             ssl_dir, 'db-client.cert')
         if not os.path.exists(cert_path):
-            log("Waiting 1m for ssl client cert validity")
+            log("Waiting 1m for ssl client cert validity", level=INFO)
             time.sleep(60)
+
         with open(cert_path, 'w') as fh:
             fh.write(b64decode(rdata['ssl_cert']))
+
         ctxt['database_ssl_cert'] = cert_path
         key_path = os.path.join(ssl_dir, 'db-client.key')
         with open(key_path, 'w') as fh:
             fh.write(b64decode(rdata['ssl_key']))
+
         ctxt['database_ssl_key'] = key_path
+
     return ctxt
 
 
@@ -253,9 +260,8 @@
     interfaces = ['identity-service']
 
     def __call__(self):
-        log('Generating template context for identity-service')
+        log('Generating template context for identity-service', level=DEBUG)
         ctxt = {}
-
         for rid in relation_ids('identity-service'):
             for unit in related_units(rid):
                 rdata = relation_get(rid=rid, unit=unit)
@@ -263,26 +269,24 @@
                 serv_host = format_ipv6_addr(serv_host) or serv_host
                 auth_host = rdata.get('auth_host')
                 auth_host = format_ipv6_addr(auth_host) or auth_host
-
-                ctxt = {
-                    'service_port': rdata.get('service_port'),
-                    'service_host': serv_host,
-                    'auth_host': auth_host,
-                    'auth_port': rdata.get('auth_port'),
-                    'admin_tenant_name': rdata.get('service_tenant'),
-                    'admin_user': rdata.get('service_username'),
-                    'admin_password': rdata.get('service_password'),
-                    'service_protocol':
-                    rdata.get('service_protocol') or 'http',
-                    'auth_protocol':
-                    rdata.get('auth_protocol') or 'http',
-                }
+                svc_protocol = rdata.get('service_protocol') or 'http'
+                auth_protocol = rdata.get('auth_protocol') or 'http'
+                ctxt = {'service_port': rdata.get('service_port'),
+                        'service_host': serv_host,
+                        'auth_host': auth_host,
+                        'auth_port': rdata.get('auth_port'),
+                        'admin_tenant_name': rdata.get('service_tenant'),
+                        'admin_user': rdata.get('service_username'),
+                        'admin_password': rdata.get('service_password'),
+                        'service_protocol': svc_protocol,
+                        'auth_protocol': auth_protocol}
                 if context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
                     # upgrading
                     ctxt['admin_tenant_id'] = rdata.get('service_tenant_id')
                     return ctxt
+
         return {}
 
 
@@ -295,21 +299,23 @@
         self.interfaces = [rel_name]
 
     def __call__(self):
-        log('Generating template context for amqp')
+        log('Generating template context for amqp', level=DEBUG)
         conf = config()
-        user_setting = 'rabbit-user'
-        vhost_setting = 'rabbit-vhost'
         if self.relation_prefix:
-            user_setting = self.relation_prefix + '-rabbit-user'
-            vhost_setting = self.relation_prefix + '-rabbit-vhost'
+            user_setting = '%s-rabbit-user' % (self.relation_prefix)
+            vhost_setting = '%s-rabbit-vhost' % (self.relation_prefix)
+        else:
+            user_setting = 'rabbit-user'
+            vhost_setting = 'rabbit-vhost'
 
         try:
             username = conf[user_setting]
             vhost = conf[vhost_setting]
         except KeyError as e:
-            log('Could not generate shared_db context. '
-                'Missing required charm config options: %s.' % e)
+            log('Could not generate shared_db context. Missing required charm '
+                'config options: %s.' % e, level=ERROR)
             raise OSContextError
+
         ctxt = {}
         for rid in relation_ids(self.rel_name):
             ha_vip_only = False
@@ -323,6 +329,7 @@
                     host = relation_get('private-address', rid=rid, unit=unit)
                     host = format_ipv6_addr(host) or host
                     ctxt['rabbitmq_host'] = host
+
                 ctxt.update({
                     'rabbitmq_user': username,
                     'rabbitmq_password': relation_get('password', rid=rid,
@@ -333,6 +340,7 @@
                 ssl_port = relation_get('ssl_port', rid=rid, unit=unit)
                 if ssl_port:
                     ctxt['rabbit_ssl_port'] = ssl_port
+
                 ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit)
                 if ssl_ca:
                     ctxt['rabbit_ssl_ca'] = ssl_ca
@@ -346,41 +354,45 @@
                 if context_complete(ctxt):
                     if 'rabbit_ssl_ca' in ctxt:
                         if not self.ssl_dir:
-                            log(("Charm not setup for ssl support "
-                                 "but ssl ca found"))
+                            log("Charm not setup for ssl support but ssl ca "
+                                "found", level=INFO)
                             break
+
                         ca_path = os.path.join(
                             self.ssl_dir, 'rabbit-client-ca.pem')
                         with open(ca_path, 'w') as fh:
                             fh.write(b64decode(ctxt['rabbit_ssl_ca']))
                             ctxt['rabbit_ssl_ca'] = ca_path
+
                     # Sufficient information found = break out!
                     break
+
             # Used for active/active rabbitmq >= grizzly
-            if ('clustered' not in ctxt or ha_vip_only) \
-                    and len(related_units(rid)) > 1:
+            if (('clustered' not in ctxt or ha_vip_only) and
+                    len(related_units(rid)) > 1):
                 rabbitmq_hosts = []
                 for unit in related_units(rid):
                     host = relation_get('private-address', rid=rid, unit=unit)
                     host = format_ipv6_addr(host) or host
                     rabbitmq_hosts.append(host)
-                ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
+
+                ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
+
         if not context_complete(ctxt):
             return {}
-        else:
-            return ctxt
+
+        return ctxt
 
 
 class CephContext(OSContextGenerator):
+    """Generates context for /etc/ceph/ceph.conf templates."""
     interfaces = ['ceph']
 
     def __call__(self):
-        '''This generates context for /etc/ceph/ceph.conf templates'''
         if not relation_ids('ceph'):
             return {}
 
-        log('Generating template context for ceph')
-
+        log('Generating template context for ceph', level=DEBUG)
         mon_hosts = []
         auth = None
         key = None
@@ -389,18 +401,18 @@
             for unit in related_units(rid):
                 auth = relation_get('auth', rid=rid, unit=unit)
                 key = relation_get('key', rid=rid, unit=unit)
-                ceph_addr = \
-                    relation_get('ceph-public-address', rid=rid, unit=unit) or \
-                    relation_get('private-address', rid=rid, unit=unit)
+                ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
+                                             unit=unit)
+                unit_priv_addr = relation_get('private-address', rid=rid,
+                                              unit=unit)
+                ceph_addr = ceph_pub_addr or unit_priv_addr
                 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
                 mon_hosts.append(ceph_addr)
 
-        ctxt = {
-            'mon_hosts': ' '.join(mon_hosts),
-            'auth': auth,
-            'key': key,
-            'use_syslog': use_syslog
-        }
+        ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
+                'auth': auth,
+                'key': key,
+                'use_syslog': use_syslog}
 
         if not os.path.isdir('/etc/ceph'):
             os.mkdir('/etc/ceph')
@@ -409,79 +421,68 @@
             return {}
 
         ensure_packages(['ceph-common'])
-
         return ctxt
 
 
-ADDRESS_TYPES = ['admin', 'internal', 'public']
-
-
 class HAProxyContext(OSContextGenerator):
+    """Provides half a context for the haproxy template, which describes
+    all peers to be included in the cluster.  Each charm needs to include
+    its own context generator that describes the port mapping.
+    """
     interfaces = ['cluster']
 
+    def __init__(self, singlenode_mode=False):
+        self.singlenode_mode = singlenode_mode
+
     def __call__(self):
-        '''
-        Builds half a context for the haproxy template, which describes
-        all peers to be included in the cluster.  Each charm needs to include
-        its own context generator that describes the port mapping.
-        '''
-        if not relation_ids('cluster'):
+        if not relation_ids('cluster') and not self.singlenode_mode:
             return {}
 
-        l_unit = local_unit().replace('/', '-')
-
         if config('prefer-ipv6'):
             addr = get_ipv6_addr(exc_list=[config('vip')])[0]
         else:
             addr = get_host_ip(unit_get('private-address'))
 
+        l_unit = local_unit().replace('/', '-')
         cluster_hosts = {}
 
         # NOTE(jamespage): build out map of configured network endpoints
         # and associated backends
         for addr_type in ADDRESS_TYPES:
-            laddr = get_address_in_network(
-                config('os-{}-network'.format(addr_type)))
+            cfg_opt = 'os-{}-network'.format(addr_type)
+            laddr = get_address_in_network(config(cfg_opt))
             if laddr:
-                cluster_hosts[laddr] = {}
-                cluster_hosts[laddr]['network'] = "{}/{}".format(
-                    laddr,
-                    get_netmask_for_address(laddr)
-                )
-                cluster_hosts[laddr]['backends'] = {}
-                cluster_hosts[laddr]['backends'][l_unit] = laddr
+                netmask = get_netmask_for_address(laddr)
+                cluster_hosts[laddr] = {'network': "{}/{}".format(laddr,
+                                                                  netmask),
+                                        'backends': {l_unit: laddr}}
                 for rid in relation_ids('cluster'):
                     for unit in related_units(rid):
-                        _unit = unit.replace('/', '-')
                         _laddr = relation_get('{}-address'.format(addr_type),
                                               rid=rid, unit=unit)
                         if _laddr:
+                            _unit = unit.replace('/', '-')
                             cluster_hosts[laddr]['backends'][_unit] = _laddr
 
         # NOTE(jamespage) no split configurations found, just use
         # private addresses
         if not cluster_hosts:
-            cluster_hosts[addr] = {}
-            cluster_hosts[addr]['network'] = "{}/{}".format(
-                addr,
-                get_netmask_for_address(addr)
-            )
-            cluster_hosts[addr]['backends'] = {}
-            cluster_hosts[addr]['backends'][l_unit] = addr
+            netmask = get_netmask_for_address(addr)
+            cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask),
+                                   'backends': {l_unit: addr}}
             for rid in relation_ids('cluster'):
                 for unit in related_units(rid):
-                    _unit = unit.replace('/', '-')
                     _laddr = relation_get('private-address',
                                           rid=rid, unit=unit)
                     if _laddr:
+                        _unit = unit.replace('/', '-')
                         cluster_hosts[addr]['backends'][_unit] = _laddr
 
-        ctxt = {
-            'frontends': cluster_hosts,
-        }
+        ctxt = {'frontends': cluster_hosts}
 
         if config('haproxy-server-timeout'):
             ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout')
+
         if config('haproxy-client-timeout'):
             ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
 
@@ -495,13 +496,18 @@
             ctxt['stat_port'] = ':8888'
 
         for frontend in cluster_hosts:
-            if len(cluster_hosts[frontend]['backends']) > 1:
+            if (len(cluster_hosts[frontend]['backends']) > 1 or
+                    self.singlenode_mode):
                 # Enable haproxy when we have enough peers.
-                log('Ensuring haproxy enabled in /etc/default/haproxy.')
+                log('Ensuring haproxy enabled in /etc/default/haproxy.',
+                    level=DEBUG)
                 with open('/etc/default/haproxy', 'w') as out:
                     out.write('ENABLED=1\n')
+
                 return ctxt
-        log('HAProxy context is incomplete, this unit has no peers.')
+
+        log('HAProxy context is incomplete, this unit has no peers.',
+            level=INFO)
         return {}
 
 
@@ -509,29 +515,28 @@
     interfaces = ['image-service']
 
     def __call__(self):
-        '''
-        Obtains the glance API server from the image-service relation.  Useful
-        in nova and cinder (currently).
-        '''
-        log('Generating template context for image-service.')
+        """Obtains the glance API server from the image-service relation.
+        Useful in nova and cinder (currently).
+        """
+        log('Generating template context for image-service.', level=DEBUG)
         rids = relation_ids('image-service')
         if not rids:
             return {}
+
         for rid in rids:
             for unit in related_units(rid):
                 api_server = relation_get('glance-api-server',
                                           rid=rid, unit=unit)
                 if api_server:
                     return {'glance_api_servers': api_server}
-        log('ImageService context is incomplete. '
-            'Missing required relation data.')
+
+        log("ImageService context is incomplete. Missing required relation "
+            "data.", level=INFO)
         return {}
 
 
 class ApacheSSLContext(OSContextGenerator):
-
-    """
-    Generates a context for an apache vhost configuration that configures
+    """Generates a context for an apache vhost configuration that configures
     HTTPS reverse proxying for one or many endpoints.  Generated context
     looks something like::
 
@@ -565,6 +570,7 @@
         else:
             cert_filename = 'cert'
             key_filename = 'key'
+
         write_file(path=os.path.join(ssl_dir, cert_filename),
                    content=b64decode(cert))
         write_file(path=os.path.join(ssl_dir, key_filename),
@@ -576,7 +582,8 @@
             install_ca_cert(b64decode(ca_cert))
 
     def canonical_names(self):
-        '''Figure out which canonical names clients will access this service'''
+        """Figure out which canonical names clients will access this service.
+        """
         cns = []
         for r_id in relation_ids('identity-service'):
             for unit in related_units(r_id):
@@ -584,7 +591,8 @@
                 for k in rdata:
                     if k.startswith('ssl_key_'):
                         cns.append(k.lstrip('ssl_key_'))
-        return list(set(cns))
+
+        return sorted(list(set(cns)))
 
     def get_network_addresses(self):
         """For each network configured, return corresponding address and vip
@@ -603,9 +611,10 @@
              ...]
         """
         addresses = []
-        vips = []
         if config('vip'):
             vips = config('vip').split()
+        else:
+            vips = []
 
         for net_type in ['os-internal-network', 'os-admin-network',
                          'os-public-network']:
@@ -614,7 +623,7 @@
             if len(vips) > 1 and is_clustered():
                 if not config(net_type):
                     log("Multiple networks configured but net_type "
-                        "is None (%s)." % net_type, level='WARNING')
+                        "is None (%s)." % net_type, level=WARNING)
                     continue
 
                 for vip in vips:
@@ -627,35 +636,35 @@
             else:
                 addresses.append((addr, addr))
 
-        return addresses
+        return sorted(addresses)
 
     def __call__(self):
-        if isinstance(self.external_ports, basestring):
+        if isinstance(self.external_ports, six.string_types):
             self.external_ports = [self.external_ports]
-        if (not self.external_ports or not https()):
+
+        if not self.external_ports or not https():
             return {}
 
         self.configure_ca()
         self.enable_modules()
 
-        ctxt = {
-            'namespace': self.service_namespace,
-            'endpoints': [],
-            'ext_ports': []
-        }
+        ctxt = {'namespace': self.service_namespace,
+                'endpoints': [],
+                'ext_ports': []}
 
         for cn in self.canonical_names():
             self.configure_cert(cn)
 
         addresses = self.get_network_addresses()
-        for address, endpoint in set(addresses):
+        for address, endpoint in sorted(set(addresses)):
             for api_port in self.external_ports:
                 ext_port = determine_apache_port(api_port)
                 int_port = determine_api_port(api_port)
                 portmap = (address, endpoint, int(ext_port), int(int_port))
                 ctxt['endpoints'].append(portmap)
                 ctxt['ext_ports'].append(int(ext_port))
-        ctxt['ext_ports'] = list(set(ctxt['ext_ports']))
+
+        ctxt['ext_ports'] = sorted(list(set(ctxt['ext_ports'])))
         return ctxt
 
 
@@ -672,21 +681,23 @@
 
     @property
     def packages(self):
-        return neutron_plugin_attribute(
-            self.plugin, 'packages', self.network_manager)
+        return neutron_plugin_attribute(self.plugin, 'packages',
+                                        self.network_manager)
 
     @property
     def neutron_security_groups(self):
         return None
 
     def _ensure_packages(self):
-        [ensure_packages(pkgs) for pkgs in self.packages]
+        for pkgs in self.packages:
+            ensure_packages(pkgs)
 
     def _save_flag_file(self):
         if self.network_manager == 'quantum':
             _file = '/etc/nova/quantum_plugin.conf'
         else:
             _file = '/etc/nova/neutron_plugin.conf'
+
         with open(_file, 'wb') as out:
             out.write(self.plugin + '\n')
 
@@ -695,13 +706,11 @@
                                           self.network_manager)
         config = neutron_plugin_attribute(self.plugin, 'config',
                                           self.network_manager)
-        ovs_ctxt = {
-            'core_plugin': driver,
-            'neutron_plugin': 'ovs',
-            'neutron_security_groups': self.neutron_security_groups,
-            'local_ip': unit_private_ip(),
-            'config': config
-        }
+        ovs_ctxt = {'core_plugin': driver,
+                    'neutron_plugin': 'ovs',
+                    'neutron_security_groups': self.neutron_security_groups,
+                    'local_ip': unit_private_ip(),
+                    'config': config}
 
         return ovs_ctxt
 
@@ -710,13 +719,11 @@
                                           self.network_manager)
         config = neutron_plugin_attribute(self.plugin, 'config',
                                           self.network_manager)
-        nvp_ctxt = {
-            'core_plugin': driver,
-            'neutron_plugin': 'nvp',
-            'neutron_security_groups': self.neutron_security_groups,
-            'local_ip': unit_private_ip(),
-            'config': config
-        }
+        nvp_ctxt = {'core_plugin': driver,
+                    'neutron_plugin': 'nvp',
+                    'neutron_security_groups': self.neutron_security_groups,
+                    'local_ip': unit_private_ip(),
+                    'config': config}
 
         return nvp_ctxt
 
@@ -726,18 +733,17 @@
         n1kv_config = neutron_plugin_attribute(self.plugin, 'config',
                                                self.network_manager)
         n1kv_user_config_flags = config('n1kv-config-flags')
-        n1kv_ctxt = {
-            'core_plugin': driver,
-            'neutron_plugin': 'n1kv',
-            'neutron_security_groups': self.neutron_security_groups,
-            'local_ip': unit_private_ip(),
-            'config': n1kv_config,
-            'vsm_ip': config('n1kv-vsm-ip'),
-            'vsm_username': config('n1kv-vsm-username'),
-            'vsm_password': config('n1kv-vsm-password'),
-            'restrict_policy_profiles': config(
-                'n1kv-restrict-policy-profiles'),
-        }
+        restrict_policy_profiles = config('n1kv-restrict-policy-profiles')
+        n1kv_ctxt = {'core_plugin': driver,
+                     'neutron_plugin': 'n1kv',
+                     'neutron_security_groups': self.neutron_security_groups,
+                     'local_ip': unit_private_ip(),
+                     'config': n1kv_config,
+                     'vsm_ip': config('n1kv-vsm-ip'),
+                     'vsm_username': config('n1kv-vsm-username'),
+                     'vsm_password': config('n1kv-vsm-password'),
+                     'restrict_policy_profiles': restrict_policy_profiles}
+
         if n1kv_user_config_flags:
             flags = config_flags_parser(n1kv_user_config_flags)
             n1kv_ctxt['user_config_flags'] = flags
@@ -749,13 +755,11 @@
                                           self.network_manager)
         config = neutron_plugin_attribute(self.plugin, 'config',
                                           self.network_manager)
-        calico_ctxt = {
-            'core_plugin': driver,
-            'neutron_plugin': 'Calico',
-            'neutron_security_groups': self.neutron_security_groups,
-            'local_ip': unit_private_ip(),
-            'config': config
-        }
+        calico_ctxt = {'core_plugin': driver,
+                       'neutron_plugin': 'Calico',
+                       'neutron_security_groups': self.neutron_security_groups,
+                       'local_ip': unit_private_ip(),
+                       'config': config}
 
         return calico_ctxt
 
@@ -764,15 +768,14 @@
             proto = 'https'
         else:
             proto = 'http'
+
         if is_clustered():
             host = config('vip')
         else:
             host = unit_get('private-address')
-        url = '%s://%s:%s' % (proto, host, '9696')
-        ctxt = {
-            'network_manager': self.network_manager,
-            'neutron_url': url,
-        }
+
+        ctxt = {'network_manager': self.network_manager,
+                'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
         return ctxt
 
     def __call__(self):
@@ -805,9 +808,7 @@
 
 
 class OSConfigFlagContext(OSContextGenerator):
-
-    """
-    Provides support for user-defined config flags.
+    """Provides support for user-defined config flags.
 
     Users can define a comma-seperated list of key=value pairs
     in the charm configuration and apply them at any point in
@@ -826,8 +827,9 @@
     def __init__(self, charm_flag='config-flags',
                  template_flag='user_config_flags'):
         """
-        charm_flag: config flags in charm configuration.
-        template_flag: insert point for user-defined flags template file.
+        :param charm_flag: config flags in charm configuration.
+        :param template_flag: insert point for user-defined flags in template
+                              file.
         """
         super(OSConfigFlagContext, self).__init__()
         self._charm_flag = charm_flag
@@ -883,7 +885,6 @@
                 },
             }
         }
-
     """
 
     def __init__(self, service, config_file, interface):
@@ -913,26 +914,28 @@
 
                     if self.service not in sub_config:
                         log('Found subordinate_config on %s but it contained'
-                            'nothing for %s service' % (rid, self.service))
+                            'nothing for %s service' % (rid, self.service),
+                            level=INFO)
                         continue
 
                     sub_config = sub_config[self.service]
                     if self.config_file not in sub_config:
                         log('Found subordinate_config on %s but it contained'
-                            'nothing for %s' % (rid, self.config_file))
+                            'nothing for %s' % (rid, self.config_file),
+                            level=INFO)
                         continue
 
                     sub_config = sub_config[self.config_file]
-                    for k, v in sub_config.iteritems():
+                    for k, v in six.iteritems(sub_config):
                         if k == 'sections':
-                            for section, config_dict in v.iteritems():
-                                log("adding section '%s'" % (section))
+                            for section, config_dict in six.iteritems(v):
+                                log("adding section '%s'" % (section),
+                                    level=DEBUG)
                                 ctxt[k][section] = config_dict
                         else:
                             ctxt[k] = v
 
         log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
-
         return ctxt
 
 
@@ -944,15 +947,14 @@
             False if config('debug') is None else config('debug')
         ctxt['verbose'] = \
             False if config('verbose') is None else config('verbose')
+
         return ctxt
 
 
 class SyslogContext(OSContextGenerator):
 
     def __call__(self):
-        ctxt = {
-            'use_syslog': config('use-syslog')
-        }
+        ctxt = {'use_syslog': config('use-syslog')}
         return ctxt
 
 
@@ -960,13 +962,9 @@
 
     def __call__(self):
         if config('prefer-ipv6'):
-            return {
-                'bind_host': '::'
-            }
+            return {'bind_host': '::'}
         else:
-            return {
-                'bind_host': '0.0.0.0'
-            }
+            return {'bind_host': '0.0.0.0'}
 
 
 class WorkerConfigContext(OSContextGenerator):
@@ -978,13 +976,12 @@
         except ImportError:
             apt_install('python-psutil', fatal=True)
             from psutil import NUM_CPUS
+
         return NUM_CPUS
 
     def __call__(self):
-        multiplier = config('worker-multiplier') or 1
-        ctxt = {
-            "workers": self.num_cpus * multiplier
-        }
+        multiplier = config('worker-multiplier') or 0
+        ctxt = {"workers": self.num_cpus * multiplier}
         return ctxt
 
 
@@ -998,22 +995,23 @@
                     for unit in related_units(rid):
                         ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
                         ctxt['zmq_host'] = relation_get('host', unit, rid)
+
         return ctxt
 
 
 class NotificationDriverContext(OSContextGenerator):
 
-    def __init__(self, zmq_relation='zeromq-configuration', amqp_relation='amqp'):
+    def __init__(self, zmq_relation='zeromq-configuration',
+                 amqp_relation='amqp'):
         """
-        :param zmq_relation   : Name of Zeromq relation to check
+        :param zmq_relation: Name of Zeromq relation to check
         """
         self.zmq_relation = zmq_relation
         self.amqp_relation = amqp_relation
 
     def __call__(self):
-        ctxt = {
-            'notifications': 'False',
-        }
+        ctxt = {'notifications': 'False'}
         if is_relation_made(self.amqp_relation):
             ctxt['notifications'] = "True"
+
         return ctxt
 
=== modified file 'hooks/charmhelpers/contrib/openstack/ip.py'
--- hooks/charmhelpers/contrib/openstack/ip.py	2014-09-22 20:21:19 +0000
+++ hooks/charmhelpers/contrib/openstack/ip.py	2014-12-11 17:56:34 +0000
@@ -2,21 +2,19 @@
     config,
     unit_get,
 )
-
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
     is_address_in_network,
     is_ipv6,
     get_ipv6_addr,
 )
-
 from charmhelpers.contrib.hahelpers.cluster import is_clustered
 
 PUBLIC = 'public'
 INTERNAL = 'int'
 ADMIN = 'admin'
 
-_address_map = {
+ADDRESS_MAP = {
     PUBLIC: {
         'config': 'os-public-network',
         'fallback': 'public-address'
@@ -33,16 +31,14 @@
 
 
 def canonical_url(configs, endpoint_type=PUBLIC):
-    '''
-    Returns the correct HTTP URL to this host given the state of HTTPS
+    """Returns the correct HTTP URL to this host given the state of HTTPS
     configuration, hacluster and charm configuration.
 
-    :configs OSTemplateRenderer: A config tempating object to inspect for
-        a complete https context.
-    :endpoint_type str: The endpoint type to resolve.
-
-    :returns str: Base URL for services on the current service unit.
-    '''
+    :param configs: OSTemplateRenderer config templating object to inspect
+                    for a complete https context.
+    :param endpoint_type: str endpoint type to resolve.
+    :param returns: str base URL for services on the current service unit.
+    """
     scheme = 'http'
     if 'https' in configs.complete_contexts():
         scheme = 'https'
@@ -53,27 +49,45 @@
 
 
 def resolve_address(endpoint_type=PUBLIC):
+    """Return unit address depending on net config.
+
+    If unit is clustered with vip(s) and has net splits defined, return vip on
+    correct network. If clustered with no nets defined, return primary vip.
+
+    If not clustered, return unit address ensuring address is on configured net
+    split if one is configured.
+
+    :param endpoint_type: Network endpoing type
+    """
     resolved_address = None
-    if is_clustered():
-        if config(_address_map[endpoint_type]['config']) is None:
-            # Assume vip is simple and pass back directly
-            resolved_address = config('vip')
+    vips = config('vip')
+    if vips:
+        vips = vips.split()
+
+    net_type = ADDRESS_MAP[endpoint_type]['config']
+    net_addr = config(net_type)
+    net_fallback = ADDRESS_MAP[endpoint_type]['fallback']
+    clustered = is_clustered()
+    if clustered:
+        if not net_addr:
+            # If no net-splits defined, we expect a single vip
+            resolved_address = vips[0]
         else:
-            for vip in config('vip').split():
-                if is_address_in_network(
-                        config(_address_map[endpoint_type]['config']),
-                        vip):
+            for vip in vips:
+                if is_address_in_network(net_addr, vip):
                     resolved_address = vip
+                    break
     else:
         if config('prefer-ipv6'):
-            fallback_addr = get_ipv6_addr(exc_list=[config('vip')])[0]
+            fallback_addr = get_ipv6_addr(exc_list=vips)[0]
         else:
-            fallback_addr = unit_get(_address_map[endpoint_type]['fallback'])
-        resolved_address = get_address_in_network(
-            config(_address_map[endpoint_type]['config']), fallback_addr)
+            fallback_addr = unit_get(net_fallback)
+
+        resolved_address = get_address_in_network(net_addr, fallback_addr)
 
     if resolved_address is None:
-        raise ValueError('Unable to resolve a suitable IP address'
-                         ' based on charm state and configuration')
-    else:
-        return resolved_address
+        raise ValueError("Unable to resolve a suitable IP address based on "
+                         "charm state and configuration. (net_type=%s, "
+                         "clustered=%s)" % (net_type, clustered))
+
+    return resolved_address
 
=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2014-11-13 04:44:29 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2014-12-11 17:56:34 +0000
@@ -14,7 +14,7 @@
 def headers_package():
     """Ensures correct linux-headers for running kernel are installed,
     for building DKMS package"""
-    kver = check_output(['uname', '-r']).strip()
+    kver = check_output(['uname', '-r']).decode('UTF-8').strip()
     return 'linux-headers-%s' % kver
 
 QUANTUM_CONF_DIR = '/etc/quantum'
@@ -22,7 +22,7 @@
 
 def kernel_version():
     """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """
-    kver = check_output(['uname', '-r']).strip()
+    kver = check_output(['uname', '-r']).decode('UTF-8').strip()
     kver = kver.split('.')
     return (int(kver[0]), int(kver[1]))
 
@@ -177,7 +177,8 @@
     elif manager == 'neutron':
         plugins = neutron_plugins()
     else:
-        log('Error: Network manager does not support plugins.')
+        log("Network manager '%s' does not support plugins." % (manager),
+            level=ERROR)
         raise Exception
 
     try:
 
=== modified file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg'
--- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-10-06 20:51:06 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-12-11 17:56:34 +0000
@@ -35,7 +35,7 @@
     stats auth admin:password
 
 {% if frontends -%}
-{% for service, ports in service_ports.iteritems() -%}
+{% for service, ports in service_ports.items() -%}
 frontend tcp-in_{{ service }}
     bind *:{{ ports[0] }}
     bind :::{{ ports[0] }}
@@ -46,7 +46,7 @@
 {% for frontend in frontends -%}
 backend {{ service }}_{{ frontend }}
     balance leastconn
-    {% for unit, address in frontends[frontend]['backends'].iteritems() -%}
+    {% for unit, address in frontends[frontend]['backends'].items() -%}
     server {{ unit }} {{ address }}:{{ ports[1] }} check
     {% endfor %}
 {% endfor -%}
 
=== modified file 'hooks/charmhelpers/contrib/openstack/templating.py'
--- hooks/charmhelpers/contrib/openstack/templating.py	2014-07-25 08:13:49 +0000
+++ hooks/charmhelpers/contrib/openstack/templating.py	2014-12-11 17:56:34 +0000
@@ -1,13 +1,13 @@
 import os
 
+import six
+
 from charmhelpers.fetch import apt_install
-
 from charmhelpers.core.hookenv import (
     log,
     ERROR,
     INFO
 )
-
 from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
 
 try:
@@ -43,7 +43,7 @@
         order by OpenStack release.
     """
     tmpl_dirs = [(rel, os.path.join(templates_dir, rel))
-                 for rel in OPENSTACK_CODENAMES.itervalues()]
+                 for rel in six.itervalues(OPENSTACK_CODENAMES)]
 
     if not os.path.isdir(templates_dir):
         log('Templates directory not found @ %s.' % templates_dir,
@@ -258,7 +258,7 @@
         """
         Write out all registered config files.
         """
-        [self.write(k) for k in self.templates.iterkeys()]
+        [self.write(k) for k in six.iterkeys(self.templates)]
 
     def set_release(self, openstack_release):
         """
@@ -275,5 +275,5 @@
         '''
         interfaces = []
         [interfaces.extend(i.complete_contexts())
-         for i in self.templates.itervalues()]
+         for i in six.itervalues(self.templates)]
         return interfaces
 
=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2014-10-24 11:36:12 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2014-12-11 17:56:34 +0000
@@ -10,11 +10,13 @@
 import socket
 import sys
 
+import six
+import yaml
+
 from charmhelpers.core.hookenv import (
     config,
     log as juju_log,
     charm_dir,
-    ERROR,
     INFO,
     relation_ids,
     relation_set
@@ -31,7 +33,8 @@
 )
 
 from charmhelpers.core.host import lsb_release, mounts, umount
-from charmhelpers.fetch import apt_install, apt_cache
+from charmhelpers.fetch import apt_install, apt_cache, install_remote
+from charmhelpers.contrib.python.packages import pip_install
 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
@@ -113,7 +116,7 @@
 
     # Best guess match based on deb string provided
     if src.startswith('deb') or src.startswith('ppa'):
-        for k, v in OPENSTACK_CODENAMES.iteritems():
+        for k, v in six.iteritems(OPENSTACK_CODENAMES):
             if v in src:
                 return v
 
@@ -134,7 +137,7 @@
 
 def get_os_version_codename(codename):
     '''Determine OpenStack version number from codename.'''
-    for k, v in OPENSTACK_CODENAMES.iteritems():
+    for k, v in six.iteritems(OPENSTACK_CODENAMES):
         if v == codename:
             return k
     e = 'Could not derive OpenStack version for '\
@@ -194,7 +197,7 @@
     else:
         vers_map = OPENSTACK_CODENAMES
 
-    for version, cname in vers_map.iteritems():
+    for version, cname in six.iteritems(vers_map):
         if cname == codename:
             return version
     # e = "Could not determine OpenStack version for package: %s" % pkg
@@ -318,7 +321,7 @@
         rc_script.write(
             "#!/bin/bash\n")
         [rc_script.write('export %s=%s\n' % (u, p))
-         for u, p in env_vars.iteritems() if u != "script_path"]
+         for u, p in six.iteritems(env_vars) if u != "script_path"]
 
 
 def openstack_upgrade_available(package):
@@ -351,8 +354,8 @@
     '''
     _none = ['None', 'none', None]
     if (block_device in _none):
-        error_out('prepare_storage(): Missing required input: '
-                  'block_device=%s.' % block_device, level=ERROR)
+        error_out('prepare_storage(): Missing required input: block_device=%s.'
+                  % block_device)
 
     if block_device.startswith('/dev/'):
         bdev = block_device
@@ -368,8 +371,7 @@
         bdev = '/dev/%s' % block_device
 
     if not is_block_device(bdev):
-        error_out('Failed to locate valid block device at %s' % bdev,
-                  level=ERROR)
+        error_out('Failed to locate valid block device at %s' % bdev)
 
     return bdev
 
@@ -418,7 +420,7 @@
 
     if isinstance(address, dns.name.Name):
         rtype = 'PTR'
-    elif isinstance(address, basestring):
+    elif isinstance(address, six.string_types):
         rtype = 'A'
     else:
         return None
@@ -486,8 +488,7 @@
               'hostname': json.dumps(hosts)}
 
     if relation_prefix:
-        keys = kwargs.keys()
-        for key in keys:
+        for key in list(kwargs.keys()):
             kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key]
             del kwargs[key]
 
@@ -508,3 +509,111 @@
             f(*args)
         return wrapped_f
     return wrap
+
+
+def git_install_requested():
+    """Returns true if openstack-origin-git is specified."""
+    return config('openstack-origin-git') != "None"
+
+
+requirements_dir = None
+
+
+def git_clone_and_install(file_name, core_project):
+    """Clone/install all OpenStack repos specified in yaml config file."""
+    global requirements_dir
+
+    if file_name == "None":
+        return
+
+    yaml_file = os.path.join(charm_dir(), file_name)
+
+    # clone/install the requirements project first
+    installed = _git_clone_and_install_subset(yaml_file,
+                                              whitelist=['requirements'])
+    if 'requirements' not in installed:
+        error_out('requirements git repository must be specified')
+
+    # clone/install all other projects except requirements and the core project
+    blacklist = ['requirements', core_project]
+    _git_clone_and_install_subset(yaml_file, blacklist=blacklist,
+                                  update_requirements=True)
+
+    # clone/install the core project
+    whitelist = [core_project]
+    installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,
+                                              update_requirements=True)
+    if core_project not in installed:
+        error_out('{} git repository must be specified'.format(core_project))
+
+
+def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],
+                                  update_requirements=False):
+    """Clone/install subset of OpenStack repos specified in yaml config file."""
+    global requirements_dir
+    installed = []
+
+    with open(yaml_file, 'r') as fd:
+        projects = yaml.load(fd)
+        for proj, val in projects.items():
+            # The project subset is chosen based on the following 3 rules:
+            # 1) If project is in blacklist, we don't clone/install it, period.
+            # 2) If whitelist is empty, we clone/install everything else.
+            # 3) If whitelist is not empty, we clone/install everything in the
+            #    whitelist.
+            if proj in blacklist:
+                continue
+            if whitelist and proj not in whitelist:
+                continue
+            repo = val['repository']
+            branch = val['branch']
+            repo_dir = _git_clone_and_install_single(repo, branch,
+                                                     update_requirements)
+            if proj == 'requirements':
+                requirements_dir = repo_dir
+            installed.append(proj)
+    return installed
+
+
+def _git_clone_and_install_single(repo, branch, update_requirements=False):
+    """Clone and install a single git repository."""
+    dest_parent_dir = "/mnt/openstack-git/"
+    dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))
+
+    if not os.path.exists(dest_parent_dir):
+        juju_log('Host dir not mounted at {}. '
+                 'Creating directory there instead.'.format(dest_parent_dir))
+        os.mkdir(dest_parent_dir)
+
+    if not os.path.exists(dest_dir):
+        juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
+        repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)
+    else:
+        repo_dir = dest_dir
+
+    if update_requirements:
+        if not requirements_dir:
+            error_out('requirements repo must be cloned before '
+                      'updating from global requirements.')
+        _git_update_requirements(repo_dir, requirements_dir)
+
+    juju_log('Installing git repo from dir: {}'.format(repo_dir))
+    pip_install(repo_dir)
+
+    return repo_dir
+
+
+def _git_update_requirements(package_dir, reqs_dir):
+    """Update from global requirements.
+
+       Update an OpenStack git directory's requirements.txt and
+       test-requirements.txt from global-requirements.txt."""
+    orig_dir = os.getcwd()
+    os.chdir(reqs_dir)
+    cmd = "python update.py {}".format(package_dir)
+    try:
+        subprocess.check_call(cmd.split(' '))
+    except subprocess.CalledProcessError:
+        package = os.path.basename(package_dir)
+        error_out("Error updating {} from global-requirements.txt".format(package))
+    os.chdir(orig_dir)
 
=== modified file 'hooks/charmhelpers/contrib/peerstorage/__init__.py'
--- hooks/charmhelpers/contrib/peerstorage/__init__.py	2014-10-06 20:53:09 +0000
+++ hooks/charmhelpers/contrib/peerstorage/__init__.py	2014-12-11 17:56:34 +0000
@@ -1,3 +1,4 @@
+import six
 from charmhelpers.core.hookenv import relation_id as current_relation_id
 from charmhelpers.core.hookenv import (
     is_relation_made,
@@ -93,7 +94,7 @@
             if ex in echo_data:
                 echo_data.pop(ex)
     else:
-        for attribute, value in rdata.iteritems():
+        for attribute, value in six.iteritems(rdata):
             for include in includes:
                 if include in attribute:
                     echo_data[attribute] = value
@@ -119,8 +120,8 @@
                  relation_settings=relation_settings,
                  **kwargs)
     if is_relation_made(peer_relation_name):
-        for key, value in dict(kwargs.items() +
-                               relation_settings.items()).iteritems():
+        for key, value in six.iteritems(dict(list(kwargs.items()) +
+                                             list(relation_settings.items()))):
             key_prefix = relation_id or current_relation_id()
             peer_store(key_prefix + delimiter + key,
                        value,
 
=== added directory 'hooks/charmhelpers/contrib/python'
=== added file 'hooks/charmhelpers/contrib/python/__init__.py'
=== added file 'hooks/charmhelpers/contrib/python/packages.py'
--- hooks/charmhelpers/contrib/python/packages.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/python/packages.py	2014-12-11 17:56:34 +0000
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+__author__ = "Jorge Niedbalski <[e-mail]>"
+
+from charmhelpers.fetch import apt_install, apt_update
+from charmhelpers.core.hookenv import log
+
+try:
+    from pip import main as pip_execute
+except ImportError:
+    apt_update()
+    apt_install('python-pip')
+    from pip import main as pip_execute
+
+
+def parse_options(given, available):
+    """Given a set of options, check if available"""
+    for key, value in sorted(given.items()):
+        if key in available:
+            yield "--{0}={1}".format(key, value)
+
+
+def pip_install_requirements(requirements, **options):
+    """Install a requirements file """
+    command = ["install"]
+
+    available_options = ('proxy', 'src', 'log', )
+    for option in parse_options(options, available_options):
+        command.append(option)
+
+    command.append("-r {0}".format(requirements))
+    log("Installing from file: {} with options: {}".format(requirements,
+                                                           command))
+    pip_execute(command)
+
+
+def pip_install(package, fatal=False, **options):
+    """Install a python package"""
+    command = ["install"]
+
+    available_options = ('proxy', 'src', 'log', "index-url", )
+    for option in parse_options(options, available_options):
+        command.append(option)
+
+    if isinstance(package, list):
+        command.extend(package)
+    else:
+        command.append(package)
+
+    log("Installing {} package with options: {}".format(package,
+                                                        command))
+    pip_execute(command)
+
+
+def pip_uninstall(package, **options):
+    """Uninstall a python package"""
+    command = ["uninstall", "-q", "-y"]
+
+    available_options = ('proxy', 'log', )
+    for option in parse_options(options, available_options):
+        command.append(option)
+
+    if isinstance(package, list):
+        command.extend(package)
+    else:
+        command.append(package)
+
+    log("Uninstalling {} package with options: {}".format(package,
+                                                          command))
+    pip_execute(command)
+
+
+def pip_list():
+    """Returns the list of current python installed packages
+    """
+    return pip_execute(["list"])
 
=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
--- hooks/charmhelpers/contrib/storage/linux/ceph.py	2014-10-24 11:36:12 +0000
+++ hooks/charmhelpers/contrib/storage/linux/ceph.py	2014-12-11 17:56:34 +0000
@@ -16,19 +16,18 @@
 from subprocess import (
     check_call,
     check_output,
-    CalledProcessError
+    CalledProcessError,
 )
-
 from charmhelpers.core.hookenv import (
     relation_get,
     relation_ids,
     related_units,
     log,
+    DEBUG,
     INFO,
     WARNING,
-    ERROR
+    ERROR,
 )
-
 from charmhelpers.core.host import (
     mount,
     mounts,
@@ -37,7 +36,6 @@
     service_running,
     umount,
 )
-
 from charmhelpers.fetch import (
     apt_install,
 )
@@ -56,99 +54,85 @@
 
 
 def install():
-    ''' Basic Ceph client installation '''
+    """Basic Ceph client installation."""
     ceph_dir = "/etc/ceph"
     if not os.path.exists(ceph_dir):
         os.mkdir(ceph_dir)
+
     apt_install('ceph-common', fatal=True)
 
 
 def rbd_exists(service, pool, rbd_img):
-    ''' Check to see if a RADOS block device exists '''
+    """Check to see if a RADOS block device exists."""
     try:
-        out = check_output(['rbd', 'list', '--id', service,
-                            '--pool', pool])
+        out = check_output(['rbd', 'list', '--id',
+                            service, '--pool', pool]).decode('UTF-8')
     except CalledProcessError:
         return False
-    else:
-        return rbd_img in out
+
+    return rbd_img in out
 
 
 def create_rbd_image(service, pool, image, sizemb):
-    ''' Create a new RADOS block device '''
-    cmd = [
-        'rbd',
-        'create',
-        image,
-        '--size',
-        str(sizemb),
-        '--id',
-        service,
-        '--pool',
-        pool
-    ]
+    """Create a new RADOS block device."""
+    cmd = ['rbd', 'create', image, '--size', str(sizemb), '--id', service,
+           '--pool', pool]
     check_call(cmd)
 
 
 def pool_exists(service, name):
-    ''' Check to see if a RADOS pool already exists '''
+    """Check to see if a RADOS pool already exists."""
     try:
-        out = check_output(['rados', '--id', service, 'lspools'])
+        out = check_output(['rados', '--id', service,
+                            'lspools']).decode('UTF-8')
     except CalledProcessError:
         return False
-    else:
-        return name in out
+
+    return name in out
 
 
 def get_osds(service):
-    '''
-    Return a list of all Ceph Object Storage Daemons
-    currently in the cluster
-    '''
+    """Return a list of all Ceph Object Storage Daemons currently in the
+    cluster.
+    """
     version = ceph_version()
     if version and version >= '0.56':
         return json.loads(check_output(['ceph', '--id', service,
-                                        'osd', 'ls', '--format=json']))
-    else:
-        return None
+                                        'osd', 'ls',
+                                        '--format=json']).decode('UTF-8'))
+
+    return None
 
 
 def create_pool(service, name, replicas=3):
-    ''' Create a new RADOS pool '''
+    """Create a new RADOS pool."""
     if pool_exists(service, name):
         log("Ceph pool {} already exists, skipping creation".format(name),
             level=WARNING)
         return
+
     # Calculate the number of placement groups based
     # on upstream recommended best practices.
     osds = get_osds(service)
     if osds:
-        pgnum = (len(osds) * 100 / replicas)
+        pgnum = (len(osds) * 100 // replicas)
     else:
         # NOTE(james-page): Default to 200 for older ceph versions
         # which don't support OSD query from cli
         pgnum = 200
-    cmd = [
-        'ceph', '--id', service,
-        'osd', 'pool', 'create',
-        name, str(pgnum)
-    ]
+
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)]
     check_call(cmd)
-    cmd = [
-        'ceph', '--id', service,
-        'osd', 'pool', 'set', name,
-        'size', str(replicas)
-    ]
+
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size',
+           str(replicas)]
     check_call(cmd)
 
 
 def delete_pool(service, name):
-    ''' Delete a RADOS pool from ceph '''
-    cmd = [
-        'ceph', '--id', service,
-        'osd', 'pool', 'delete',
-        name, '--yes-i-really-really-mean-it'
-    ]
+    """Delete a RADOS pool from ceph."""
+    cmd = ['ceph', '--id', service, 'osd', 'pool', 'delete', name,
+           '--yes-i-really-really-mean-it']
     check_call(cmd)
 
 
@@ -161,44 +145,43 @@
 
 
 def create_keyring(service, key):
-    ''' Create a new Ceph keyring containing key'''
+    """Create a new Ceph keyring containing key."""
     keyring = _keyring_path(service)
     if os.path.exists(keyring):
-        log('ceph: Keyring exists at %s.' % keyring, level=WARNING)
+        log('Ceph keyring exists at %s.' % keyring, level=WARNING)
         return
-    cmd = [
-        'ceph-authtool',
-        keyring,
-        '--create-keyring',
-        '--name=client.{}'.format(service),
-        '--add-key={}'.format(key)
-    ]
+
+    cmd = ['ceph-authtool', keyring, '--create-keyring',
+           '--name=client.{}'.format(service), '--add-key={}'.format(key)]
     check_call(cmd)
-    log('ceph: Created new ring at %s.' % keyring, level=INFO)
+    log('Created new ceph keyring at %s.' % keyring, level=DEBUG)
 
 
 def create_key_file(service, key):
-    ''' Create a file containing key '''
+    """Create a file containing key."""
     keyfile = _keyfile_path(service)
     if os.path.exists(keyfile):
-        log('ceph: Keyfile exists at %s.' % keyfile, level=WARNING)
+        log('Keyfile exists at %s.' % keyfile, level=WARNING)
         return
+
     with open(keyfile, 'w') as fd:
         fd.write(key)
-    log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
+
+    log('Created new keyfile at %s.' % keyfile, level=INFO)
 
 
 def get_ceph_nodes():
-    ''' Query named relation 'ceph' to detemine current nodes '''
+    """Query named relation 'ceph' to determine current nodes."""
     hosts = []
     for r_id in relation_ids('ceph'):
         for unit in related_units(r_id):
             hosts.append(relation_get('private-address', unit=unit, rid=r_id))
+
     return hosts
 
 
 def configure(service, key, auth, use_syslog):
-    ''' Perform basic configuration of Ceph '''
+    """Perform basic configuration of Ceph."""
     create_keyring(service, key)
     create_key_file(service, key)
     hosts = get_ceph_nodes()
@@ -211,17 +194,17 @@
 
 
 def image_mapped(name):
-    ''' Determine whether a RADOS block device is mapped locally '''
+    """Determine whether a RADOS block device is mapped locally."""
     try:
-        out = check_output(['rbd', 'showmapped'])
+        out = check_output(['rbd', 'showmapped']).decode('UTF-8')
     except CalledProcessError:
         return False
-    else:
-        return name in out
+
+    return name in out
 
 
 def map_block_storage(service, pool, image):
-    ''' Map a RADOS block device for local use '''
+    """Map a RADOS block device for local use."""
     cmd = [
         'rbd',
         'map',
@@ -235,31 +218,32 @@
 
 
 def filesystem_mounted(fs):
-    ''' Determine whether a filesytems is already mounted '''
+    """Determine whether a filesytems is already mounted."""
     return fs in [f for f, m in mounts()]
 
 
 def make_filesystem(blk_device, fstype='ext4', timeout=10):
-    ''' Make a new filesystem on the specified block device '''
+    """Make a new filesystem on the specified block device."""
     count = 0
     e_noent = os.errno.ENOENT
     while not os.path.exists(blk_device):
         if count >= timeout:
-            log('ceph: gave up waiting on block device %s' % blk_device,
+            log('Gave up waiting on block device %s' % blk_device,
                 level=ERROR)
             raise IOError(e_noent, os.strerror(e_noent), blk_device)
-        log('ceph: waiting for block device %s to appear' % blk_device,
-            level=INFO)
+
+        log('Waiting for block device %s to appear' % blk_device,
+            level=DEBUG)
         count += 1
         time.sleep(1)
     else:
-        log('ceph: Formatting block device %s as filesystem %s.' %
+        log('Formatting block device %s as filesystem %s.' %
             (blk_device, fstype), level=INFO)
         check_call(['mkfs', '-t', fstype, blk_device])
 
 
 def place_data_on_block_device(blk_device, data_src_dst):
-    ''' Migrate data in data_src_dst to blk_device and then remount '''
+    """Migrate data in data_src_dst to blk_device and then remount."""
     # mount block device into /mnt
     mount(blk_device, '/mnt')
     # copy data to /mnt
@@ -279,8 +263,8 @@
 
 # TODO: re-use
 def modprobe(module):
-    ''' Load a kernel module and configure for auto-load on reboot '''
-    log('ceph: Loading kernel module', level=INFO)
+    """Load a kernel module and configure for auto-load on reboot."""
+    log('Loading kernel module', level=INFO)
     cmd = ['modprobe', module]
     check_call(cmd)
     with open('/etc/modules', 'r+') as modules:
@@ -289,7 +273,7 @@
 
 
 def copy_files(src, dst, symlinks=False, ignore=None):
-    ''' Copy files from src to dst '''
+    """Copy files from src to dst."""
     for item in os.listdir(src):
         s = os.path.join(src, item)
         d = os.path.join(dst, item)
@@ -302,8 +286,7 @@
 def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
                         blk_device, fstype, system_services=[],
                         replicas=3):
-    """
-    NOTE: This function must only be called from a single service unit for
+    """NOTE: This function must only be called from a single service unit for
     the same rbd_img otherwise data loss will occur.
 
     Ensures given pool and RBD image exists, is mapped to a block device,
@@ -317,15 +300,16 @@
     """
     # Ensure pool, RBD image, RBD mappings are in place.
     if not pool_exists(service, pool):
-        log('ceph: Creating new pool {}.'.format(pool))
+        log('Creating new pool {}.'.format(pool), level=INFO)
         create_pool(service, pool, replicas=replicas)
 
     if not rbd_exists(service, pool, rbd_img):
-        log('ceph: Creating RBD image ({}).'.format(rbd_img))
+        log('Creating RBD image ({}).'.format(rbd_img), level=INFO)
         create_rbd_image(service, pool, rbd_img, sizemb)
 
     if not image_mapped(rbd_img):
-        log('ceph: Mapping RBD Image {} as a Block Device.'.format(rbd_img))
+        log('Mapping RBD Image {} as a Block Device.'.format(rbd_img),
+            level=INFO)
         map_block_storage(service, pool, rbd_img)
 
     # make file system
@@ -340,45 +324,47 @@
 
         for svc in system_services:
             if service_running(svc):
-                log('ceph: Stopping services {} prior to migrating data.'
-                    .format(svc))
+                log('Stopping services {} prior to migrating data.'
+                    .format(svc), level=DEBUG)
                 service_stop(svc)
 
         place_data_on_block_device(blk_device, mount_point)
 
         for svc in system_services:
-            log('ceph: Starting service {} after migrating data.'
-                .format(svc))
+            log('Starting service {} after migrating data.'
+                .format(svc), level=DEBUG)
             service_start(svc)
 
 
 def ensure_ceph_keyring(service, user=None, group=None):
-    '''
-    Ensures a ceph keyring is created for a named service
-    and optionally ensures user and group ownership.
+    """Ensures a ceph keyring is created for a named service and optionally
+    ensures user and group ownership.
 
     Returns False if no ceph key is available in relation state.
-    '''
+    """
     key = None
     for rid in relation_ids('ceph'):
         for unit in related_units(rid):
             key = relation_get('key', rid=rid, unit=unit)
             if key:
                 break
+
     if not key:
         return False
+
     create_keyring(service=service, key=key)
     keyring = _keyring_path(service)
     if user and group:
         check_call(['chown', '%s.%s' % (user, group), keyring])
+
     return True
 
 
 def ceph_version():
-    ''' Retrieve the local version of ceph '''
+    """Retrieve the local version of ceph."""
     if os.path.exists('/usr/bin/ceph'):
         cmd = ['ceph', '-v']
-        output = check_output(cmd)
+        output = check_output(cmd).decode('US-ASCII')
         output = output.split()
         if len(output) > 3:
             return output[2]
 
=== modified file 'hooks/charmhelpers/contrib/storage/linux/loopback.py'
--- hooks/charmhelpers/contrib/storage/linux/loopback.py	2014-03-27 10:54:38 +0000
+++ hooks/charmhelpers/contrib/storage/linux/loopback.py	2014-12-11 17:56:34 +0000
@@ -1,12 +1,12 @@
-
 import os
 import re
-
 from subprocess import (
     check_call,
     check_output,
 )
 
+import six
+
 
 ##################################################
 # loopback device helpers.
@@ -37,7 +37,7 @@
     '''
     file_path = os.path.abspath(file_path)
     check_call(['losetup', '--find', file_path])
-    for d, f in loopback_devices().iteritems():
+    for d, f in six.iteritems(loopback_devices()):
         if f == file_path:
             return d
 
@@ -51,7 +51,7 @@
 
     :returns: str: Full path to the ensured loopback device (eg, /dev/loop0)
     '''
-    for d, f in loopback_devices().iteritems():
+    for d, f in six.iteritems(loopback_devices()):
         if f == path:
             return d
 
 
=== modified file 'hooks/charmhelpers/contrib/storage/linux/lvm.py'
--- hooks/charmhelpers/contrib/storage/linux/lvm.py	2014-05-19 11:42:30 +0000
+++ hooks/charmhelpers/contrib/storage/linux/lvm.py	2014-12-11 17:56:34 +0000
@@ -61,6 +61,7 @@
     vg = None
     pvd = check_output(['pvdisplay', block_device]).splitlines()
     for l in pvd:
+        l = l.decode('UTF-8')
         if l.strip().startswith('VG Name'):
             vg = ' '.join(l.strip().split()[2:])
     return vg
 
=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2014-09-15 07:44:00 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-12-11 17:56:34 +0000
@@ -30,7 +30,8 @@
     # sometimes sgdisk exits non-zero; this is OK, dd will clean up
     call(['sgdisk', '--zap-all', '--mbrtogpt',
           '--clear', block_device])
-    dev_end = check_output(['blockdev', '--getsz', block_device])
+    dev_end = check_output(['blockdev', '--getsz',
+                            block_device]).decode('UTF-8')
     gpt_end = int(dev_end.split()[0]) - 100
     check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
                 'bs=1M', 'count=1'])
@@ -47,7 +48,7 @@
         it doesn't.
     '''
     is_partition = bool(re.search(r".*[0-9]+\b", device))
-    out = check_output(['mount'])
+    out = check_output(['mount']).decode('UTF-8')
     if is_partition:
         return bool(re.search(device + r"\b", out))
     return bool(re.search(device + r"[0-9]+\b", out))
 
=== modified file 'hooks/charmhelpers/contrib/unison/__init__.py'
--- hooks/charmhelpers/contrib/unison/__init__.py	2014-03-27 10:54:38 +0000
+++ hooks/charmhelpers/contrib/unison/__init__.py	2014-12-11 17:56:34 +0000
@@ -185,13 +185,14 @@
         relation_set(ssh_authorized_hosts=authed_hosts)
 
 
-def _run_as_user(user):
+def _run_as_user(user, gid=None):
     try:
         user = pwd.getpwnam(user)
     except KeyError:
         log('Invalid user: %s' % user)
         raise Exception
-    uid, gid = user.pw_uid, user.pw_gid
+    uid = user.pw_uid
+    gid = gid or user.pw_gid
     os.environ['HOME'] = user.pw_dir
 
     def _inner():
@@ -200,8 +201,8 @@
     return _inner
 
 
-def run_as_user(user, cmd):
-    return check_output(cmd, preexec_fn=_run_as_user(user), cwd='/')
+def run_as_user(user, cmd, gid=None):
+    return check_output(cmd, preexec_fn=_run_as_user(user, gid), cwd='/')
 
 
 def collect_authed_hosts(peer_interface):
@@ -227,8 +228,8 @@
     return hosts
 
 
-def sync_path_to_host(path, host, user, verbose=False):
-    cmd = copy(BASE_CMD)
+def sync_path_to_host(path, host, user, verbose=False, cmd=None, gid=None):
+    cmd = cmd or copy(BASE_CMD)
     if not verbose:
         cmd.append('-silent')
 
@@ -241,17 +242,23 @@
 
     try:
         log('Syncing local path %s to %s@%s:%s' % (path, user, host, path))
-        run_as_user(user, cmd)
+        run_as_user(user, cmd, gid)
     except:
         log('Error syncing remote files')
 
 
-def sync_to_peer(host, user, paths=[], verbose=False):
+def sync_to_peer(host, user, paths=None, verbose=False, cmd=None, gid=None):
     '''Sync paths to an specific host'''
-    [sync_path_to_host(p, host, user, verbose) for p in paths]
-
-
-def sync_to_peers(peer_interface, user, paths=[], verbose=False):
+    if paths:
+        for p in paths:
+            sync_path_to_host(p, host, user, verbose, cmd, gid)
+
+
+def sync_to_peers(peer_interface, user, paths=None,
+                  verbose=False, cmd=None, gid=None):
     '''Sync all hosts to an specific path'''
-    for host in collect_authed_hosts(peer_interface):
-        sync_to_peer(host, user, paths, verbose)
+    '''The type of group is integer, it allows user has permissions to '''
+    '''operate a directory have a different group id with the user id.'''
+    if paths:
+        for host in collect_authed_hosts(peer_interface):
+            sync_to_peer(host, user, paths, verbose, cmd, gid)
 
=== modified file 'hooks/charmhelpers/core/fstab.py'
--- hooks/charmhelpers/core/fstab.py	2014-06-24 17:11:12 +0000
+++ hooks/charmhelpers/core/fstab.py	2014-12-11 17:56:34 +0000
@@ -3,10 +3,11 @@
 
 __author__ = 'Jorge Niedbalski R. <[e-mail]>'
 
+import io
 import os
 
 
-class Fstab(file):
+class Fstab(io.FileIO):
     """This class extends file in order to implement a file reader/writer
     for file `/etc/fstab`
     """
@@ -24,8 +25,8 @@
                 options = "defaults"
 
             self.options = options
-            self.d = d
-            self.p = p
+            self.d = int(d)
+            self.p = int(p)
 
         def __eq__(self, o):
             return str(self) == str(o)
@@ -45,7 +46,7 @@
             self._path = path
         else:
             self._path = self.DEFAULT_PATH
-        file.__init__(self, self._path, 'r+')
+        super(Fstab, self).__init__(self._path, 'rb+')
 
     def _hydrate_entry(self, line):
         # NOTE: use split with no arguments to split on any
@@ -58,8 +59,9 @@
     def entries(self):
         self.seek(0)
         for line in self.readlines():
+            line = line.decode('us-ascii')
             try:
-                if not line.startswith("#"):
+                if line.strip() and not line.startswith("#"):
                     yield self._hydrate_entry(line)
             except ValueError:
                 pass
@@ -75,14 +77,14 @@
         if self.get_entry_by_attr('device', entry.device):
             return False
 
-        self.write(str(entry) + '\n')
+        self.write((str(entry) + '\n').encode('us-ascii'))
         self.truncate()
         return entry
 
     def remove_entry(self, entry):
         self.seek(0)
 
-        lines = self.readlines()
+        lines = [l.decode('us-ascii') for l in self.readlines()]
 
         found = False
         for index, line in enumerate(lines):
@@ -97,7 +99,7 @@
         lines.remove(line)
 
         self.seek(0)
-        self.write(''.join(lines))
+        self.write(''.join(lines).encode('us-ascii'))
         self.truncate()
         return True
 
 
=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2014-10-24 11:36:12 +0000
+++ hooks/charmhelpers/core/hookenv.py	2014-12-11 17:56:34 +0000
@@ -9,9 +9,14 @@
 import yaml
 import subprocess
 import sys
-import UserDict
 from subprocess import CalledProcessError
 
+import six
+if not six.PY3:
+    from UserDict import UserDict
+else:
+    from collections import UserDict
+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@@ -63,16 +68,18 @@
     command = ['juju-log']
     if level:
         command += ['-l', level]
+    if not isinstance(message, six.string_types):
+        message = repr(message)
     command += [message]
     subprocess.call(command)
 
 
-class Serializable(UserDict.IterableUserDict):
+class Serializable(UserDict):
     """Wrapper, an object that can be serialized to yaml or json"""
 
     def __init__(self, obj):
         # wrap the object
-        UserDict.IterableUserDict.__init__(self)
+        UserDict.__init__(self)
         self.data = obj
 
     def __getattr__(self, attr):
@@ -218,7 +225,7 @@
         prev_keys = []
         if self._prev_dict is not None:
             prev_keys = self._prev_dict.keys()
-        return list(set(prev_keys + dict.keys(self)))
+        return list(set(prev_keys + list(dict.keys(self))))
 
     def load_previous(self, path=None):
         """Load previous copy of config from disk.
@@ -269,7 +276,7 @@
 
         """
         if self._prev_dict:
-            for k, v in self._prev_dict.iteritems():
+            for k, v in six.iteritems(self._prev_dict):
                 if k not in self:
                     self[k] = v
         with open(self.path, 'w') as f:
@@ -284,7 +291,8 @@
         config_cmd_line.append(scope)
     config_cmd_line.append('--format=json')
     try:
-        config_data = json.loads(subprocess.check_output(config_cmd_line))
+        config_data = json.loads(
+            subprocess.check_output(config_cmd_line).decode('UTF-8'))
         if scope is not None:
             return config_data
         return Config(config_data)
@@ -303,10 +311,10 @@
     if unit:
         _args.append(unit)
     try:
-        return json.loads(subprocess.check_output(_args))
+        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
     except ValueError:
         return None
-    except CalledProcessError, e:
+    except CalledProcessError as e:
         if e.returncode == 2:
             return None
         raise
@@ -318,7 +326,7 @@
     relation_cmd_line = ['relation-set']
     if relation_id is not None:
         relation_cmd_line.extend(('-r', relation_id))
-    for k, v in (relation_settings.items() + kwargs.items()):
+    for k, v in (list(relation_settings.items()) + list(kwargs.items())):
         if v is None:
             relation_cmd_line.append('{}='.format(k))
         else:
@@ -335,7 +343,8 @@
     relid_cmd_line = ['relation-ids', '--format=json']
     if reltype is not None:
         relid_cmd_line.append(reltype)
-        return json.loads(subprocess.check_output(relid_cmd_line)) or []
+        return json.loads(
+            subprocess.check_output(relid_cmd_line).decode('UTF-8')) or []
     return []
 
 
@@ -346,7 +355,8 @@
     units_cmd_line = ['relation-list', '--format=json']
     if relid is not None:
         units_cmd_line.extend(('-r', relid))
-    return json.loads(subprocess.check_output(units_cmd_line)) or []
+    return json.loads(
+        subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
 
 
 @cached
@@ -386,21 +396,31 @@
 
 
 @cached
+def metadata():
+    """Get the current charm metadata.yaml contents as a python object"""
+    with open(os.path.join(charm_dir(), 'metadata.yaml')) as md:
+        return yaml.safe_load(md)
+
+
+@cached
 def relation_types():
     """Get a list of relation types supported by this charm"""
-    charmdir = os.environ.get('CHARM_DIR', '')
-    mdf = open(os.path.join(charmdir, 'metadata.yaml'))
-    md = yaml.safe_load(mdf)
     rel_types = []
+    md = metadata()
     for key in ('provides', 'requires', 'peers'):
         section = md.get(key)
         if section:
             rel_types.extend(section.keys())
-    mdf.close()
     return rel_types
 
 
 @cached
+def charm_name():
+    """Get the name of the current charm as is specified on metadata.yaml"""
+    return metadata().get('name')
+
+
+@cached
 def relations():
     """Get a nested dictionary of relation data for all related units"""
     rels = {}
@@ -455,7 +475,7 @@
     """Get the unit ID for the remote unit"""
     _args = ['unit-get', '--format=json', attribute]
     try:
-        return json.loads(subprocess.check_output(_args))
+        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
     except ValueError:
         return None
 
 
=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-10-24 11:36:12 +0000
+++ hooks/charmhelpers/core/host.py	2014-12-11 17:56:34 +0000
@@ -14,11 +14,12 @@
 import subprocess
 import hashlib
 from contextlib import contextmanager
-
 from collections import OrderedDict
 
-from hookenv import log
-from fstab import Fstab
+import six
+
+from .hookenv import log
+from .fstab import Fstab
 
 
 def service_start(service_name):
@@ -54,7 +55,9 @@
 def service_running(service):
     """Determine whether a system service is running"""
     try:
-        output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT)
+        output = subprocess.check_output(
+            ['service', service, 'status'],
+            stderr=subprocess.STDOUT).decode('UTF-8')
     except subprocess.CalledProcessError:
         return False
     else:
@@ -67,7 +70,9 @@
 def service_available(service_name):
     """Determine whether a system service is available"""
     try:
-        subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
+        subprocess.check_output(
+            ['service', service_name, 'status'],
+            stderr=subprocess.STDOUT).decode('UTF-8')
     except subprocess.CalledProcessError as e:
         return 'unrecognized service' not in e.output
     else:
@@ -96,6 +101,26 @@
     return user_info
 
 
+def add_group(group_name, system_group=False):
+    """Add a group to the system"""
+    try:
+        group_info = grp.getgrnam(group_name)
+        log('group {0} already exists!'.format(group_name))
+    except KeyError:
+        log('creating group {0}'.format(group_name))
+        cmd = ['addgroup']
+        if system_group:
+            cmd.append('--system')
+        else:
+            cmd.extend([
+                '--group',
+            ])
+        cmd.append(group_name)
+        subprocess.check_call(cmd)
+        group_info = grp.getgrnam(group_name)
+    return group_info
+
+
 def add_user_to_group(username, group):
     """Add a user to a group"""
     cmd = [
@@ -115,7 +140,7 @@
     cmd.append(from_path)
     cmd.append(to_path)
     log(" ".join(cmd))
-    return subprocess.check_output(cmd).strip()
+    return subprocess.check_output(cmd).decode('UTF-8').strip()
 
 
 def symlink(source, destination):
@@ -130,7 +155,7 @@
     subprocess.check_call(cmd)
 
 
-def mkdir(path, owner='root', group='root', perms=0555, force=False):
+def mkdir(path, owner='root', group='root', perms=0o555, force=False):
     """Create a directory"""
     log("Making dir {} {}:{} {:o}".format(path, owner, group,
                                           perms))
@@ -146,7 +171,7 @@
     os.chown(realpath, uid, gid)
 
 
-def write_file(path, content, owner='root', group='root', perms=0444):
+def write_file(path, content, owner='root', group='root', perms=0o444):
     """Create or overwrite a file with the contents of a string"""
     log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
     uid = pwd.getpwnam(owner).pw_uid
@@ -177,7 +202,7 @@
     cmd_args.extend([device, mountpoint])
     try:
         subprocess.check_output(cmd_args)
-    except subprocess.CalledProcessError, e:
+    except subprocess.CalledProcessError as e:
         log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
         return False
 
@@ -191,7 +216,7 @@
     cmd_args = ['umount', mountpoint]
     try:
         subprocess.check_output(cmd_args)
-    except subprocess.CalledProcessError, e:
+    except subprocess.CalledProcessError as e:
         log('Error unmounting {}\n{}'.format(mountpoint, e.output))
         return False
 
@@ -218,8 +243,8 @@
     """
     if os.path.exists(path):
         h = getattr(hashlib, hash_type)()
-        with open(path, 'r') as source:
-            h.update(source.read())  # IGNORE:E1101 - it does have update
+        with open(path, 'rb') as source:
+            h.update(source.read())
         return h.hexdigest()
     else:
         return None
@@ -297,7 +322,7 @@
     if length is None:
         length = random.choice(range(35, 45))
     alphanumeric_chars = [
-        l for l in (string.letters + string.digits)
+        l for l in (string.ascii_letters + string.digits)
         if l not in 'l0QD1vAEIOUaeiou']
     random_chars = [
         random.choice(alphanumeric_chars) for _ in range(length)]
@@ -306,14 +331,14 @@
 
 def list_nics(nic_type):
     '''Return a list of nics of given type(s)'''
-    if isinstance(nic_type, basestring):
+    if isinstance(nic_type, six.string_types):
         int_types = [nic_type]
     else:
         int_types = nic_type
     interfaces = []
     for int_type in int_types:
         cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
-        ip_output = subprocess.check_output(cmd).split('\n')
+        ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
         ip_output = (line for line in ip_output if line)
         for line in ip_output:
             if line.split()[1].startswith(int_type):
@@ -335,7 +360,7 @@
 
 def get_nic_mtu(nic):
     cmd = ['ip', 'addr', 'show', nic]
-    ip_output = subprocess.check_output(cmd).split('\n')
+    ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
     mtu = ""
     for line in ip_output:
         words = line.split()
@@ -346,7 +371,7 @@
 
 def get_nic_hwaddr(nic):
     cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
-    ip_output = subprocess.check_output(cmd)
+    ip_output = subprocess.check_output(cmd).decode('UTF-8')
     hwaddr = ""
     words = ip_output.split()
     if 'link/ether' in words:
@@ -363,8 +388,8 @@
 
     '''
     import apt_pkg
-    from charmhelpers.fetch import apt_cache
     if not pkgcache:
+        from charmhelpers.fetch import apt_cache
         pkgcache = apt_cache()
     pkg = pkgcache[package]
     return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
 
=== modified file 'hooks/charmhelpers/core/services/helpers.py'
--- hooks/charmhelpers/core/services/helpers.py	2014-10-06 20:51:06 +0000
+++ hooks/charmhelpers/core/services/helpers.py	2014-12-11 17:56:34 +0000
@@ -196,7 +196,7 @@
         if not os.path.isabs(file_name):
             file_name = os.path.join(hookenv.charm_dir(), file_name)
         with open(file_name, 'w') as file_stream:
-            os.fchmod(file_stream.fileno(), 0600)
+            os.fchmod(file_stream.fileno(), 0o600)
             yaml.dump(config_data, file_stream)
 
     def read_context(self, file_name):
@@ -211,15 +211,19 @@
 
 class TemplateCallback(ManagerCallback):
     """
-    Callback class that will render a Jinja2 template, for use as a ready action.
-
-    :param str source: The template source file, relative to `$CHARM_DIR/templates`
+    Callback class that will render a Jinja2 template, for use as a ready
+    action.
+
+    :param str source: The template source file, relative to
+    `$CHARM_DIR/templates`
+
     :param str target: The target to write the rendered template to
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
     """
-    def __init__(self, source, target, owner='root', group='root', perms=0444):
+    def __init__(self, source, target,
+                 owner='root', group='root', perms=0o444):
         self.source = source
         self.target = target
         self.owner = owner
 
=== modified file 'hooks/charmhelpers/core/templating.py'
--- hooks/charmhelpers/core/templating.py	2014-08-13 13:11:49 +0000
+++ hooks/charmhelpers/core/templating.py	2014-12-11 17:56:34 +0000
@@ -4,7 +4,8 @@
 from charmhelpers.core import hookenv
 
 
-def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None):
+def render(source, target, context, owner='root', group='root',
+           perms=0o444, templates_dir=None):
     """
     Render a template.
 
 
=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-10-24 11:36:12 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-12-11 17:56:34 +0000
@@ -5,10 +5,6 @@
 from charmhelpers.core.host import (
     lsb_release
 )
-from urlparse import (
-    urlparse,
-    urlunparse,
-)
 import subprocess
 from charmhelpers.core.hookenv import (
     config,
@@ -16,6 +12,12 @@
 )
 import os
 
+import six
+if six.PY3:
+    from urllib.parse import urlparse, urlunparse
+else:
+    from urlparse import urlparse, urlunparse
+
 
 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
@@ -149,7 +151,7 @@
     cmd = ['apt-get', '--assume-yes']
     cmd.extend(options)
     cmd.append('install')
-    if isinstance(packages, basestring):
+    if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
@@ -182,7 +184,7 @@
 def apt_purge(packages, fatal=False):
     """Purge one or more packages"""
     cmd = ['apt-get', '--assume-yes', 'purge']
-    if isinstance(packages, basestring):
+    if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
@@ -193,7 +195,7 @@
 def apt_hold(packages, fatal=False):
     """Hold one or more packages"""
     cmd = ['apt-mark', 'hold']
-    if isinstance(packages, basestring):
+    if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
@@ -260,7 +262,7 @@
 
     if key:
         if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
-            with NamedTemporaryFile() as key_file:
+            with NamedTemporaryFile('w+') as key_file:
                 key_file.write(key)
                 key_file.flush()
                 key_file.seek(0)
@@ -297,14 +299,14 @@
     sources = safe_load((config(sources_var) or '').strip()) or []
     keys = safe_load((config(keys_var) or '').strip()) or None
 
-    if isinstance(sources, basestring):
+    if isinstance(sources, six.string_types):
         sources = [sources]
 
     if keys is None:
         for source in sources:
             add_source(source, None)
     else:
-        if isinstance(keys, basestring):
+        if isinstance(keys, six.string_types):
             keys = [keys]
 
         if len(sources) != len(keys):
@@ -401,7 +403,7 @@
         while result is None or result == APT_NO_LOCK:
             try:
                 result = subprocess.check_call(cmd, env=env)
-            except subprocess.CalledProcessError, e:
+            except subprocess.CalledProcessError as e:
                 retry_count = retry_count + 1
                 if retry_count > APT_NO_LOCK_RETRY_COUNT:
                     raise
 
=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
--- hooks/charmhelpers/fetch/archiveurl.py	2014-10-06 20:51:06 +0000
+++ hooks/charmhelpers/fetch/archiveurl.py	2014-12-11 17:56:34 +0000
@@ -1,8 +1,23 @@
 import os
-import urllib2
-from urllib import urlretrieve
-import urlparse
 import hashlib
+import re
+
+import six
+if six.PY3:
+    from urllib.request import (
+        build_opener, install_opener, urlopen, urlretrieve,
+        HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+    )
+    from urllib.parse import urlparse, urlunparse, parse_qs
+    from urllib.error import URLError
+else:
+    from urllib import urlretrieve
+    from urllib2 import (
+        build_opener, install_opener, urlopen,
+        HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+        URLError
+    )
+    from urlparse import urlparse, urlunparse, parse_qs
 
 from charmhelpers.fetch import (
     BaseFetchHandler,
@@ -15,6 +30,24 @@
 from charmhelpers.core.host import mkdir, check_hash
 
 
+def splituser(host):
+    '''urllib.splituser(), but six's support of this seems broken'''
+    _userprog = re.compile('^(.*)@(.*)$')
+    match = _userprog.match(host)
+    if match:
+        return match.group(1, 2)
+    return None, host
+
+
+def splitpasswd(user):
+    '''urllib.splitpasswd(), but six's support of this is missing'''
+    _passwdprog = re.compile('^([^:]*):(.*)$', re.S)
+    match = _passwdprog.match(user)
+    if match:
+        return match.group(1, 2)
+    return user, None
+
+
 class ArchiveUrlFetchHandler(BaseFetchHandler):
     """
     Handler to download archive files from arbitrary URLs.
@@ -42,20 +75,20 @@
         """
         # propogate all exceptions
         # URLError, OSError, etc
-        proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+        proto, netloc, path, params, query, fragment = urlparse(source)
         if proto in ('http', 'https'):
-            auth, barehost = urllib2.splituser(netloc)
+            auth, barehost = splituser(netloc)
             if auth is not None:
-                source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
-                username, password = urllib2.splitpasswd(auth)
-                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+                source = urlunparse((proto, barehost, path, params, query, fragment))
+                username, password = splitpasswd(auth)
+                passman = HTTPPasswordMgrWithDefaultRealm()
                 # Realm is set to None in add_password to force the username and password
                 # to be used whatever the realm
                 passman.add_password(None, source, username, password)
-                authhandler = urllib2.HTTPBasicAuthHandler(passman)
-                opener = urllib2.build_opener(authhandler)
-                urllib2.install_opener(opener)
-        response = urllib2.urlopen(source)
+                authhandler = HTTPBasicAuthHandler(passman)
+                opener = build_opener(authhandler)
+                install_opener(opener)
+        response = urlopen(source)
         try:
             with open(dest, 'w') as dest_file:
                 dest_file.write(response.read())
@@ -91,17 +124,21 @@
         url_parts = self.parse_url(source)
         dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
         if not os.path.exists(dest_dir):
-            mkdir(dest_dir, perms=0755)
+            mkdir(dest_dir, perms=0o755)
         dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
         try:
             self.download(source, dld_file)
-        except urllib2.URLError as e:
+        except URLError as e:
             raise UnhandledSource(e.reason)
         except OSError as e:
             raise UnhandledSource(e.strerror)
-        options = urlparse.parse_qs(url_parts.fragment)
+        options = parse_qs(url_parts.fragment)
         for key, value in options.items():
-            if key in hashlib.algorithms:
+            if not six.PY3:
+                algorithms = hashlib.algorithms
+            else:
+                algorithms = hashlib.algorithms_available
+            if key in algorithms:
                 check_hash(dld_file, value, key)
         if checksum:
             check_hash(dld_file, checksum, hash_type)
 
=== modified file 'hooks/charmhelpers/fetch/bzrurl.py'
--- hooks/charmhelpers/fetch/bzrurl.py	2014-06-24 17:11:12 +0000
+++ hooks/charmhelpers/fetch/bzrurl.py	2014-12-11 17:56:34 +0000
@@ -5,6 +5,10 @@
 )
 from charmhelpers.core.host import mkdir
 
+import six
+if six.PY3:
+    raise ImportError('bzrlib does not support Python3')
+
 try:
     from bzrlib.branch import Branch
 except ImportError:
@@ -42,7 +46,7 @@
         dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
                                 branch_name)
         if not os.path.exists(dest_dir):
-            mkdir(dest_dir, perms=0755)
+            mkdir(dest_dir, perms=0o755)
         try:
             self.branch(source, dest_dir)
         except OSError as e:
 
=== modified file 'hooks/charmhelpers/fetch/giturl.py'
--- hooks/charmhelpers/fetch/giturl.py	2014-10-24 11:36:12 +0000
+++ hooks/charmhelpers/fetch/giturl.py	2014-12-11 17:56:34 +0000
@@ -5,6 +5,10 @@
 )
 from charmhelpers.core.host import mkdir
 
+import six
+if six.PY3:
+    raise ImportError('GitPython does not support Python 3')
+
 try:
     from git import Repo
 except ImportError:
@@ -17,7 +21,7 @@
     """Handler for git branches via generic and github URLs"""
     def can_handle(self, source):
         url_parts = self.parse_url(source)
-        #TODO (mattyw) no support for ssh git@ yet
+        # TODO (mattyw) no support for ssh git@ yet
         if url_parts.scheme not in ('http', 'https', 'git'):
             return False
         else:
@@ -30,13 +34,16 @@
         repo = Repo.clone_from(source, dest)
         repo.git.checkout(branch)
 
-    def install(self, source, branch="master"):
+    def install(self, source, branch="master", dest=None):
         url_parts = self.parse_url(source)
         branch_name = url_parts.path.strip("/").split("/")[-1]
-        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
-                                branch_name)
+        if dest:
+            dest_dir = os.path.join(dest, branch_name)
+        else:
+            dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+                                    branch_name)
         if not os.path.exists(dest_dir):
-            mkdir(dest_dir, perms=0755)
+            mkdir(dest_dir, perms=0o755)
         try:
             self.clone(source, dest_dir, branch)
         except OSError as e:
 
=== modified file 'tests/charmhelpers/__init__.py'
--- tests/charmhelpers/__init__.py	2014-06-24 17:11:12 +0000
+++ tests/charmhelpers/__init__.py	2014-12-11 17:56:34 +0000
@@ -0,0 +1,22 @@
+# Bootstrap charm-helpers, installing its dependencies if necessary using
+# only standard libraries.
+import subprocess
+import sys
+
+try:
+    import six  # flake8: noqa
+except ImportError:
+    if sys.version_info.major == 2:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
+    else:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
+    import six  # flake8: noqa
+
+try:
+    import yaml  # flake8: noqa
+except ImportError:
+    if sys.version_info.major == 2:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
+    else:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
+    import yaml  # flake8: noqa
 
=== modified file 'tests/charmhelpers/contrib/amulet/deployment.py'
--- tests/charmhelpers/contrib/amulet/deployment.py	2014-10-06 20:51:06 +0000
+++ tests/charmhelpers/contrib/amulet/deployment.py	2014-12-11 17:56:34 +0000
@@ -1,6 +1,6 @@
 import amulet
-
 import os
+import six
 
 
 class AmuletDeployment(object):
@@ -52,12 +52,12 @@
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""
-        for k, v in relations.iteritems():
+        for k, v in six.iteritems(relations):
             self.d.relate(k, v)
 
     def _configure_services(self, configs):
         """Configure all of the services."""
-        for service, config in configs.iteritems():
+        for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
     def _deploy(self):
 
=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
--- tests/charmhelpers/contrib/amulet/utils.py	2014-07-30 15:11:18 +0000
+++ tests/charmhelpers/contrib/amulet/utils.py	2014-12-11 17:56:34 +0000
@@ -5,6 +5,8 @@
 import sys
 import time
 
+import six
+
 
 class AmuletUtils(object):
     """Amulet utilities.
@@ -58,7 +60,7 @@
            Verify the specified services are running on the corresponding
            service units.
            """
-        for k, v in commands.iteritems():
+        for k, v in six.iteritems(commands):
             for cmd in v:
                 output, code = k.run(cmd)
                 if code != 0:
@@ -100,11 +102,11 @@
            longs, or can be a function that evaluate a variable and returns a
            bool.
            """
-        for k, v in expected.iteritems():
+        for k, v in six.iteritems(expected):
             if k in actual:
-                if (isinstance(v, basestring) or
+                if (isinstance(v, six.string_types) or
                         isinstance(v, bool) or
-                        isinstance(v, (int, long))):
+                        isinstance(v, six.integer_types)):
                     if v != actual[k]:
                         return "{}:{}".format(k, actual[k])
                 elif not v(actual[k]):
 
=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-10-06 20:51:06 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-12-11 17:56:34 +0000
@@ -1,3 +1,4 @@
+import six
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -69,7 +70,7 @@
 
     def _configure_services(self, configs):
         """Configure all of the services."""
-        for service, config in configs.iteritems():
+        for service, config in six.iteritems(configs):
             self.d.configure(service, config)
 
     def _get_openstack_release(self):
 
=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-10-06 20:51:06 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-12-11 17:56:34 +0000
@@ -7,6 +7,8 @@
 import keystoneclient.v2_0 as keystone_client
 import novaclient.v1_1.client as nova_client
 
+import six
+
 from charmhelpers.contrib.amulet.utils import (
     AmuletUtils
 )
@@ -60,7 +62,7 @@
            expected service catalog endpoints.
            """
         self.log.debug('actual: {}'.format(repr(actual)))
-        for k, v in expected.iteritems():
+        for k, v in six.iteritems(expected):
             if k in actual:
                 ret = self._validate_dict_data(expected[k][0], actual[k][0])
                 if ret: